Compare commits

...
Author SHA1 Message Date
Jack Robison 3fff5ccb49 use array.array to serialize TX_COUNTS_STATE 2020-12-01 18:47:16 -05:00
Jack Robison aa0f49633a off by one 2020-12-01 18:47:16 -05:00
Jack Robison ede167bb56 initialize TX_COUNTS_STATE from existing TX_COUNT_PREFIX 2020-12-01 18:47:16 -05:00
Jack Robison c03b1b5a93 fix block_txs iterator stop condition 2020-12-01 17:00:35 -05:00
Jack Robison 68a97b0e61 remove dead code 2020-12-01 17:00:03 -05:00
Jack Robison f86d940f20 TX_COUNTS_STATE 2020-12-01 11:30:14 -05:00
Jack Robison 314f8b0d42 fix uncaught error in chris45 test 2020-12-01 11:30:14 -05:00
Jack Robison 5ec8f355d4 cleanup 2020-12-01 11:30:14 -05:00
Jack Robison f46bf0924a update dockerfile 2020-12-01 10:11:13 -05:00
Jack Robison 6cbe559778 constants 2020-11-30 14:40:17 -05:00
Jack Robison 4ab2d7e624 ci 2020-11-30 14:40:09 -05:00
Jack Robison a82abb33ac fix 2020-11-30 14:26:04 -05:00
Jack Robison 70e5ce4806 rocksdb 2020-11-30 09:49:01 -05:00
Jack Robison d960ba7412 combine leveldb databases 2020-11-30 09:49:01 -05:00
Jack Robison 18340c248d merge tx_db and headers_db 2020-11-30 09:47:38 -05:00
Jack Robison 9012db0cfb use rocksdb by default 2020-11-29 14:30:02 -05:00
Jack Robison 7143b475a1 RocksReaderContext 2020-11-29 14:30:02 -05:00
Jack Robison 976387fefb RocksDBIterator 2020-11-29 14:30:02 -05:00
Jack Robison 1c01faed28 add rocksdb requirement, update dockerfile 2020-11-29 13:36:28 -05:00
Jack Robison df3254b371 Merkle staticmethods 2020-11-29 13:35:43 -05:00
Jack Robison 9b4afe9816 store txids in memory, faster address subscription/history 2020-11-25 16:30:11 -05:00
Jack Robison 23bb5598d5 Merge pull request #3095 from lbryio/fix-transaction-getbatch-order
ensure transactions are returned in the order they're requested
2020-11-24 17:21:12 -05:00
Jack Robison af1d7813e9 ensure transactions are returned in the order they're requested 2020-11-24 11:52:07 -05:00
Jack Robison 16c2e5a585 Merge pull request #3093 from lbryio/fix-merkle
fix off-by-one when getting block txs for tx_merkle
2020-11-23 19:08:11 -05:00
Jack Robison c02750edbd fix tx_merkle 2020-11-23 16:48:24 -05:00
Jack Robison 7204ddafec Merge pull request #3092 from lbryio/faster-transaction-getbatch
faster `blockchain.transaction.get_batch`
2020-11-23 12:29:14 -05:00
Jack Robison faeba9a7e4 faster blockchain.transaction.get_batch 2020-11-23 12:05:47 -05:00
Jack Robison 190d238a1f faster read_headers 2020-11-23 12:03:11 -05:00
Jack Robison 715451b5fb v0.85.0 2020-11-21 00:14:20 -05:00
Jack Robison 87f1895405 Merge pull request #3090 from lbryio/fast-headers
store headers in memory on wallet server
2020-11-20 23:33:18 -05:00
Jack Robison 923d817751 store headers in memory 2020-11-20 22:37:42 -05:00
Jack Robison 0728209b66 Merge pull request #3089 from lbryio/force_checkpoint
Ensure no client .wal files remain on shutdown
2020-11-20 21:12:33 -05:00
Victor Shyba b8b9dcc2ee WAL_CHECKPOINT on close 2020-11-20 20:16:03 -03:00
Jack Robison f35e879852 Merge pull request #3088 from lbryio/batched-inflate-outputs
fix flooding with resolve/claim search
2020-11-20 17:43:56 -05:00
Jack Robison 34f4f12eb9 pylint 2020-11-20 16:15:57 -05:00
Jack Robison fa63bf758d delete single_call_context, use session pool 2020-11-20 15:52:11 -05:00
Jack Robison f6b396ae64 Merge branch 'fix-reorg' 2020-11-20 13:32:59 -05:00
Jack Robison 2c7fd58e34 threaded compress headers 2020-11-20 13:09:54 -05:00
Jack Robison 982f2c9634 fix test 2020-11-20 10:57:28 -05:00
Jack Robison f2fd42b47a use write_batch 2020-11-19 18:37:49 -05:00
Jack Robison 1b4ccad938 allow settings TRENDING_ALGORITHMS to nothing 2020-11-19 18:36:34 -05:00
Jack Robison a9de1ce8e0 fix fs_transactions 2020-11-19 12:40:46 -05:00
Jack Robison ac752d5ec2 Merge pull request #3082 from lbryio/faster-leveldb
store transactions for wallet server in leveldb instead of requesting from lbrycrd
2020-11-17 17:02:11 -05:00
Jack Robison 632d8d02d2 remove dead code 2020-11-17 16:17:24 -05:00
Jack Robison 48aeb26e02 threaded_get_merkle
-remove dead code
2020-11-17 16:05:08 -05:00
Jack Robison 1694af8b5e get transactions from leveldb instead of lbrycrd rpc 2020-11-17 15:30:48 -05:00
Jack Robison 83bcab9cd2 cleanup 2020-11-17 15:30:48 -05:00
Jack Robison bdc7f4b3f5 combine tx_count_db and hashes_db, add tx db 2020-11-17 15:30:48 -05:00
Lex Berezhny 39202a3d79 old sequence resolution test 2020-11-17 11:24:29 -05:00
Lex Berezhny 912065a121 remove url sequence feature from code and tests 2020-11-17 11:24:29 -05:00
Thomas Zarebczan c8466afac2 Remove * from invalid characters
This is already part of other URLs and channels, will be removed from spec as it won't be used for sequencing.
2020-11-16 14:09:04 -05:00
Victor Shyba 2619e162c1 recover invalid state described on #3026 2020-11-09 16:03:38 -05:00
Lex Berezhny e1112e17f8 v0.84.0 2020-11-08 19:42:33 -05:00
Jack Robison 92b2ead74c Merge pull request #3078 from lbryio/null-language
support `none` argument to `any_languages` filter in `claim_search`
2020-11-05 22:09:11 -05:00
Jack Robison bbed9b94c1 null language filtering for claim search 2020-11-05 21:23:49 -05:00
Jack Robison 73d07311db Merge branch 'brendon' 2020-11-05 21:21:24 -05:00
Brendon J. Brewer 1cdff47477 Rewrite of variable_decay.py for speed improvements 2020-11-05 21:15:55 -05:00
Jack Robison 511a5c3f82 Merge pull request #3058 from lbryio/faster-resolve
faster resolve and claim_search
2020-11-05 21:09:07 -05:00
Jack Robison 853885e2ff debug 2020-11-05 19:25:34 -05:00
Jack Robison d83936a66a fix uncaught error 2020-11-04 22:21:35 -05:00
Jack Robison 5517d2bf56 fix new_sdk_server arg for resolve 2020-11-03 16:23:31 -05:00
Jack Robison f21ab49ac5 bump aioupnp requirement 2020-11-03 10:39:38 -05:00
Jack Robison 925a458abe tags 2020-11-03 10:39:08 -05:00
Jack Robison 76946c447f use single_call_context for claim_search and resolve 2020-11-03 10:39:08 -05:00
Jack Robison 2faa29b1c4 fix dht_monitor script 2020-11-03 10:39:08 -05:00
Lex Berezhny 6826cc311d update test to use new url spec 2020-10-30 11:37:54 -04:00
Lex Berezhny 5e17ce0a0b minor fixup 2020-10-30 11:37:54 -04:00
Kevin Raoofi e8d299d3b6 Allow : or # for claim_id
This removes the code for trying multiple patterns and the setup for it

Added a few unit tests to check that the parsed URL is as expected
2020-10-30 11:37:54 -04:00
Kevin Raoofi 7637aa2ab6 Added support for spec compliant URL parsing
Legacy URLs are preserved by attempting to parse the new URL format and,
on failing that, it'll attempt the legacy one.

Tests had to be updated such that the correct things are asserted
against each other.
2020-10-30 11:37:54 -04:00
Jack Robison ab067d1d3a fix unclean closing of blockchain.db 2020-10-26 16:23:28 -04:00
Lex Berezhny e6f84666c7 added glorious data in dictionary doc for support_sum 2020-10-16 12:03:01 -04:00
Lex Berezhny 4c5429af15 fix lint 2020-10-16 12:03:01 -04:00
Alex Grintsvayg 0a0ac3b7c9 pass-through for new support_sum api 2020-10-16 12:03:01 -04:00
Lex Berezhny 24833ce9fb v0.83.0 2020-10-10 13:03:42 -04:00
Lex Berezhny ec2c18dc87 regenerate docs 2020-10-10 13:02:38 -04:00
jessop 7384609e74 support comment pinning 2020-10-10 13:02:38 -04:00
Lex Berezhny 3047649650 v0.82.0 2020-09-29 08:49:16 -04:00
kodxana d298dac3f3 Fixed docker-compose - no snapshot error
Curently new containers do not get provide snapshots from LBRYinc (Wallet and CRD) that causes containers to get into restart loop.
2020-09-29 08:43:49 -04:00
Lex Berezhny 1574bca8a8 Merge pull request #3054 from lbryio/ios-support
iOS support
2020-09-29 08:42:17 -04:00
Lex Berezhny ec2f6c6b80 lint 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola 838cc60161 conditional import 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola 310c61a5cc check KIVY_BUILD env variable 2020-09-29 07:52:33 -04:00
Lex Berezhny 318cc15323 Merge pull request #3045 from lbryio/bencode-byte-keys
add forward compatibility for byte datagram keys
2020-09-29 07:23:49 -04:00
Jack Robison 3a64ceb4d6 add forward compatibility for byte datagram keys 2020-09-28 15:56:13 -04:00
Lex Berezhny d0f21c0095 Merge pull request #3050 from lbryio/language-indexes
add indexes for `any_languages` argument to `claim_search`
2020-09-28 15:52:41 -04:00
Jack Robison 46dc15dd29 use language indexes 2020-09-28 14:45:51 -04:00
Jack Robison 8dc654b513 add language table and indexes 2020-09-28 14:45:51 -04:00
Jack Robison 7000ac3f3f update indexed tags 2020-09-28 14:45:51 -04:00
Lex Berezhny 43c2e8d8e9 Merge pull request #3051 from lbryio/comments-v2
Comments v2 with Reactions
2020-09-28 14:44:54 -04:00
jessop 0231139b01 support anonymous react_list 2020-09-27 16:36:51 -04:00
jessop d6ee6446dd react list takes comment_ids 2020-09-26 15:21:26 -04:00
Lex Berezhny 7b666efcf8 lbrycrd download path 2020-09-24 20:11:40 -04:00
Lex Berezhny eb5d2198fc lint 2020-09-24 20:09:46 -04:00
Lex Berezhny 34e44ebd1c minor cleanup and lint fixes 2020-09-24 20:03:22 -04:00
jessop bf2f4bc040 wip 2020-09-24 17:47:59 -04:00
jessop 9dc4559aba new comment and reactions api 2020-09-23 16:43:28 -04:00
Lex Berezhny eba8856261 v0.81.0 2020-08-19 15:47:53 -04:00
Lex Berezhny a8c5aa471a update docs 2020-08-19 15:46:29 -04:00
Lex Berezhny 52f6dcf092 Merge pull request #3021 from lbryio/debug-reflector
Improve reflector upload cancellation handling
2020-08-19 15:44:27 -04:00
Jack Robison dec79f3742 pylint 2020-08-19 15:24:17 -04:00
Jack Robison 8bdcac0f3e close reader before deleting blob 2020-08-19 15:24:17 -04:00
Jack Robison 8426b674a3 improve reflector upload cancellation handling 2020-08-19 15:24:17 -04:00
Lex Berezhny da391bcc8d Merge pull request #3028 from lbryio/limit_claims_per_channel
added `--limit_claims_per_channel` argument to `claim_search` to only return up to the specified number of claims per channel
2020-08-19 15:22:53 -04:00
Lex Berezhny 2d7443d454 slight cleanup 2020-08-19 13:59:26 -04:00
Lex Berezhny 991987ed76 docopt fix 2020-08-19 12:08:41 -04:00
Lex Berezhny ec24ebf2cf added --limit_claims_per_channel argument to claim_search to only return up to the specified number of claims per channel 2020-08-19 10:51:31 -04:00
Lex Berezhny 6ba0976085 Merge pull request #3022 from btzr-io/patch-2
Add webvtt mime type ( subtitles, captions )
2020-08-12 18:58:07 -04:00
Baltazar Gomez 2b88d01a01 Add webvtt mime type ( subtitles, captions ) 2020-08-12 02:25:01 -05:00
Lex Berezhny 0c09f24cbf v0.80.0 2020-08-04 16:53:52 -04:00
Lex Berezhny 61d22afeba Merge pull request #3016 from lbryio/release_all_outputs_on_start
release reserved outputs on startup
2020-08-04 16:53:15 -04:00
Lex Berezhny 9f1ed6e8c3 release_all_outputs on startup 2020-08-04 15:03:58 -04:00
Lex Berezhny bbc4113cac Merge pull request #3015 from lbryio/new_sdk_forwarding
added support for using the new SDK for `resolve` and `claim_search` by passing `--new_sdk_server`
2020-08-04 15:03:39 -04:00
Lex Berezhny 91194bf422 doc fix 2020-08-04 14:12:40 -04:00
Lex Berezhny 9c5f940b00 claim search forwarding to new sdk 2020-08-04 12:11:02 -04:00
Lex Berezhny 455b4043b8 new resolve 2020-08-04 11:33:39 -04:00
45 changed files with 2867 additions and 1294 deletions

No files matched your search

+3 -1
View File
@@ -6,10 +6,12 @@
/.coverage*
/lbry-venv
/venv
/lbry/blockchain
lbry.egg-info
__pycache__
_trial_temp/
trending*.log
/tests/integration/blockchain/files
/tests/.coverage.*
@@ -17,4 +19,4 @@ _trial_temp/
/lbry/wallet/bin
/.vscode
/.gitignore
/.gitignore
+11 -1
View File
@@ -28,24 +28,32 @@ stages:
test:lint:
stage: test
script:
- apt-get update
- apt-get install -y liblz4-dev libsnappy-dev librocksdb-dev
- make install tools
- make lint
test:unit:
stage: test
script:
- apt-get update
- apt-get install -y liblz4-dev libsnappy-dev librocksdb-dev
- make install tools
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
test:datanetwork-integration:
stage: test
script:
- apt-get update
- apt-get install -y liblz4-dev libsnappy-dev librocksdb-dev
- pip install tox-travis
- tox -e datanetwork --recreate
test:blockchain-integration:
stage: test
script:
- apt-get update
- apt-get install -y liblz4-dev libsnappy-dev librocksdb-dev
- pip install tox-travis
- tox -e blockchain
@@ -53,13 +61,15 @@ test:other-integration:
stage: test
script:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- apt-get install -y --no-install-recommends ffmpeg liblz4-dev libsnappy-dev librocksdb-dev
- pip install tox-travis
- tox -e other
test:json-api:
stage: test
script:
- apt-get update
- apt-get install -y liblz4-dev libsnappy-dev librocksdb-dev
- make install tools
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
+7 -1
View File
@@ -10,6 +10,10 @@ ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
RUN apt-get update && \
apt-get -y --no-install-recommends install \
liblz4-dev \
libbz2-dev \
libsnappy-dev \
librocksdb-dev \
wget \
tar unzip \
build-essential \
@@ -31,7 +35,8 @@ RUN chown -R $user:$user $projects_dir
USER $user
WORKDIR $projects_dir
RUN pip install uvloop
RUN pip install uvloop cython
RUN pip install python-rocksdb
RUN make install
RUN python3 docker/set_build.py
RUN rm ~/.cache -rf
@@ -48,5 +53,6 @@ ENV DB_DIRECTORY=$db_dir
ENV MAX_SESSIONS=1000000000
ENV MAX_SEND=1000000000000000000
ENV EVENT_LOOP_POLICY=uvloop
ENV DB_ENGINE=rocksdb
COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+4 -2
View File
@@ -16,7 +16,8 @@ services:
- "lbrycrd:/data/.lbrycrd"
environment:
- RUN_MODE=default
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
# Curently not snapshot provided
#- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
- RPC_ALLOW_IP=0.0.0.0/0
wallet_server:
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
@@ -30,5 +31,6 @@ services:
volumes:
- "wallet_server:/database"
environment:
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
# Curently not snapshot provided
# - SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
+475 -167
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.79.1"
__version__ = "0.85.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+1 -1
View File
@@ -337,9 +337,9 @@ class BlobFile(AbstractBlob):
return self.loop.create_task(write_blob())
def delete(self):
super().delete()
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
@classmethod
async def create_from_unencrypted(
+1 -1
View File
@@ -611,7 +611,7 @@ class Config(CLIConfig):
('lbrynet4.lbry.com', 4444) # ASIA
])
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
+17 -7
View File
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
self.response = response.decode()
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
def _decode_datagram(datagram: bytes):
msg_types = {
REQUEST_TYPE: RequestDatagram,
RESPONSE_TYPE: ResponseDatagram,
@@ -152,19 +152,29 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
}
primitive: typing.Dict = bdecode(datagram)
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
converted = {
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
}
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
else:
raise ValueError("invalid datagram type")
datagram_class = msg_types[datagram_type]
decoded = {
k: primitive[i] # pylint: disable=unsubscriptable-object
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
for i, k in enumerate(datagram_class.required_fields)
if i in primitive # pylint: disable=unsupported-membership-test
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
}
for i, _ in enumerate(OPTIONAL_FIELDS):
if i + OPTIONAL_ARG_OFFSET in primitive:
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
return decoded, datagram_class
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
decoded, datagram_class = _decode_datagram(datagram)
return datagram_class(**decoded)
+15 -5
View File
@@ -22,10 +22,10 @@ def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
if isinstance(channel, Output):
try:
signing_field = comment['comment_id'] if abandon else comment['comment']
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
pieces = [
comment['signing_ts'].encode(),
cid2hash(comment['channel_id']),
@@ -41,9 +41,9 @@ def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
return False
def sign_comment(comment: dict, channel: Output, abandon=False):
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if abandon else comment['comment']
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
@@ -52,11 +52,21 @@ def sign_comment(comment: dict, channel: Output, abandon=False):
'signing_ts': timestamp
})
def sign_reaction(reaction: dict, channel: Output):
timestamp = str(int(time.time()))
signing_field = reaction['channel_name']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
reaction.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
async with utils.aiohttp_request('POST', url, json=json_body) as response:
try:
result = await response.json()
+215 -10
View File
@@ -557,6 +557,14 @@ class Daemon(metaclass=JSONRPCServerType):
await self.component_manager.stop()
else:
self.component_startup_task.cancel()
# the wallet component might have not started
try:
wallet_component = self.component_manager.get_actual_component('wallet')
except NameError:
pass
else:
await wallet_component.stop()
await self.component_manager.stop()
log.info("stopped api components")
await self.rpc_runner.cleanup()
await self.streaming_runner.cleanup()
@@ -986,10 +994,12 @@ class Daemon(metaclass=JSONRPCServerType):
[--include_sent_supports]
[--include_sent_tips]
[--include_received_tips]
[--new_sdk_server=<new_sdk_server>]
Options:
--urls=<urls> : (str, list) one or more urls to resolve
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase reciepts
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
--include_purchase_receipt : (bool) lookup and include a receipt if this wallet
has purchased the claim being resolved
--include_is_my_output : (bool) lookup and include a boolean indicating
@@ -2293,6 +2303,41 @@ class Daemon(metaclass=JSONRPCServerType):
kwargs['is_not_spent'] = True
return self.jsonrpc_txo_list(**kwargs)
async def jsonrpc_support_sum(self, claim_id, new_sdk_server, include_channel_content=False, **kwargs):
"""
List total staked supports for a claim, grouped by the channel that signed the support.
+
+ If claim_id is a channel claim, you can use --include_channel_content to also include supports for
+ content claims in the channel.
!!!! NOTE: PAGINATION DOES NOT DO ANYTHING AT THE MOMENT !!!!!
Usage:
support_sum <claim_id> <new_sdk_server>
[--include_channel_content]
[--page=<page>] [--page_size=<page_size>]
Options:
--claim_id=<claim_id> : (str) claim id
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
--include_channel_content : (bool) if claim_id is for a channel, include supports for claims in
that channel
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
Returns: {Paginated[Dict]}
"""
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
support_sums = await self.ledger.sum_supports(
new_sdk_server, claim_id=claim_id, include_channel_content=include_channel_content, **kwargs
)
return {
"items": support_sums,
"page": page_num,
"page_size": page_size
}
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_search(self, **kwargs):
"""
@@ -2308,6 +2353,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--channel=<channel> |
[[--channel_ids=<channel_ids>...] [--not_channel_ids=<not_channel_ids>...]]]
[--has_channel_signature] [--valid_channel_signature | --invalid_channel_signature]
[--limit_claims_per_channel=<limit_claims_per_channel>]
[--is_controlling] [--release_time=<release_time>] [--public_key_id=<public_key_id>]
[--timestamp=<timestamp>] [--creation_timestamp=<creation_timestamp>]
[--height=<height>] [--creation_height=<creation_height>]
@@ -2327,6 +2373,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--not_locations=<not_locations>...]
[--order_by=<order_by>...] [--page=<page>] [--page_size=<page_size>]
[--wallet_id=<wallet_id>] [--include_purchase_receipt] [--include_is_my_output]
[--new_sdk_server=<new_sdk_server>]
Options:
--name=<name> : (str) claim name (normalized)
@@ -2355,6 +2402,8 @@ class Daemon(metaclass=JSONRPCServerType):
--invalid_channel_signature : (bool) claims with invalid channel signature or no signature,
use in conjunction with --has_channel_signature to
only get claims with invalid signatures
--limit_claims_per_channel=<limit_claims_per_channel>: (int) only return up to the specified
number of claims per channel
--is_controlling : (bool) winning claims of their respective name
--public_key_id=<public_key_id> : (str) only return channels having this public key id, this is
the same key as used in the wallet file to map
@@ -2432,6 +2481,7 @@ class Daemon(metaclass=JSONRPCServerType):
has purchased the claim
--include_is_my_output : (bool) lookup and include a boolean indicating
if claim being resolved is yours
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
Returns: {Paginated[Output]}
"""
@@ -5041,8 +5091,9 @@ class Daemon(metaclass=JSONRPCServerType):
if hidden ^ visible:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_claim_hidden_comments',
'comment.List',
claim_id=claim_id,
visible=visible,
hidden=hidden,
page=page,
page_size=page_size
@@ -5050,7 +5101,7 @@ class Daemon(metaclass=JSONRPCServerType):
else:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_claim_comments',
'comment.List',
claim_id=claim_id,
parent_id=parent_id,
page=page,
@@ -5102,8 +5153,10 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"is_pinned": (boolean) Channel owner has pinned this comment,
"signature": (str) The signature of the comment,
"signing_ts": (str) The timestamp used to sign the comment,
"channel_url": (str) Channel's URI in the ClaimTrie,
@@ -5125,7 +5178,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Create', comment_body)
response.update({
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
})
@@ -5151,18 +5204,20 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"signature": (str) The signature of the comment,
"signing_ts": (str) Timestamp used to sign the most recent signature,
"channel_url": (str) Channel's URI in the ClaimTrie,
"is_pinned": (boolean) Channel owner has pinned this comment,
"parent_id": (str) Comment this is replying to, (None) if this is the root,
"timestamp": (int) The time at which comment was entered into the server at, in nanoseconds.
}
"""
channel = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_channel_from_comment_id',
'comment.GetChannelFromCommentID',
comment_id=comment_id
)
if 'error' in channel:
@@ -5179,7 +5234,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(edited_comment, channel_claim)
return await comment_client.jsonrpc_post(
self.conf.comment_server, 'edit_comment', edited_comment
self.conf.comment_server, 'comment.Edit', edited_comment
)
@requires(WALLET_COMPONENT)
@@ -5205,7 +5260,7 @@ class Daemon(metaclass=JSONRPCServerType):
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
abandon_comment_body = {'comment_id': comment_id}
channel = await comment_client.jsonrpc_post(
self.conf.comment_server, 'get_channel_from_comment_id', comment_id=comment_id
self.conf.comment_server, 'comment.GetChannelFromCommentID', comment_id=comment_id
)
if 'error' in channel:
return {comment_id: {'abandoned': False}}
@@ -5214,8 +5269,8 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(abandon_comment_body, channel, abandon=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'abandon_comment', abandon_comment_body)
comment_client.sign_comment(abandon_comment_body, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Abandon', abandon_comment_body)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_hide(self, comment_ids: typing.Union[str, list], wallet_id=None):
@@ -5259,9 +5314,159 @@ class Daemon(metaclass=JSONRPCServerType):
for_signing=True
)
piece = {'comment_id': comment['comment_id']}
comment_client.sign_comment(piece, channel, abandon=True)
comment_client.sign_comment(piece, channel, sign_comment_id=True)
pieces.append(piece)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'hide_comments', pieces=pieces)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Hide', pieces=pieces)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_pin(self, comment_id=None, channel_id=None, channel_name=None,
channel_account_id=None, remove=False, wallet_id=None):
"""
Pin a comment published to a claim you control.
Usage:
comment_pin (<comment_id> | --comment_id=<comment_id>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
[--remove]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_id=<comment_id> : (str) Hash identifying the comment to pin
--channel_id=<claim_id> : (str) The ID of channel owning the commented claim
--channel_name=<claim_name> : (str) The name of channel owning the commented claim
--remove : (bool) remove the pin
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--wallet_id=<wallet_id : (str) restrict operation to specific wallet
Returns: lists containing the ids comments that are hidden and visible.
{
"hidden": (list) IDs of hidden comments.
"visible": (list) IDs of visible comments.
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
comment_pin_args = {
'comment_id': comment_id,
'channel_name': channel_name,
'channel_id': channel_id,
'remove': remove,
}
comment_client.sign_comment(comment_pin_args, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Pin', comment_pin_args)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, remove=False, clear_types=None, react_type=None, wallet_id=None):
"""
Create and associate a reaction emoji with a comment using your channel identity.
Usage:
comment_react (--comment_ids=<comment_ids>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
(--react_type=<react_type>)
[(--remove) | (--clear_types=<clear_types>)]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_ids=<comment_ids> : (str) one or more comment id reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--react_type=<react_type> : (str) name of reaction type
--remove : (bool) remove specified react_type
--clear_types=<clear_types> : (str) types to clear when adding another type
Returns:
(dict) Reaction object if successfully made, (None) otherwise
{
"Reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction
...
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_body = {
'comment_ids': comment_ids,
'channel_id': channel_id,
'channel_name': channel.claim_name,
'type': react_type,
'remove': remove,
'clear_types': clear_types,
}
comment_client.sign_reaction(react_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.React', react_body)
return response
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react_list(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, react_types=None, wallet_id=None):
"""
List reactions emoji with a claim using your channel identity.
Usage:
comment_react_list (--comment_ids=<comment_ids>)
[(--channel_id=<channel_id>)(--channel_name=<channel_name>)]
[--react_types=<react_types>]
Options:
--comment_ids=<comment_ids> : (str) The comment ids reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--react_types=<react_type> : (str) comma delimited reaction types
Returns:
(dict) Comment object if successfully made, (None) otherwise
{
"my_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
"other_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
react_list_body = {
'comment_ids': comment_ids,
}
if channel_id:
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_list_body['channel_id'] = channel_id
react_list_body['channel_name'] = channel.claim_name
if react_types:
react_list_body['types'] = react_types
if channel_id:
comment_client.sign_reaction(react_list_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.List', react_list_body)
return response
async def broadcast_or_release(self, tx, blocking=False):
await self.wallet_manager.broadcast_or_release(tx, blocking)
+1
View File
@@ -123,6 +123,7 @@ types_map = {
'.txt': ('text/plain', 'document'),
'.ustar': ('application/x-ustar', 'binary'),
'.vcf': ('text/x-vcard', 'document'),
'.vtt': ('text/vtt', 'document'),
'.wav': ('audio/x-wav', 'audio'),
'.webm': ('video/webm', 'video'),
'.wiz': ('application/msword', 'document'),
+9 -2
View File
@@ -25,12 +25,14 @@ def set_reference(reference, claim_hash, rows):
class Censor:
__slots__ = 'streams', 'channels', 'censored', 'total'
__slots__ = 'streams', 'channels', 'limit_claims_per_channel', 'censored', 'claims_in_channel', 'total'
def __init__(self, streams: dict = None, channels: dict = None):
def __init__(self, streams: dict = None, channels: dict = None, limit_claims_per_channel: int = None):
self.streams = streams or {}
self.channels = channels or {}
self.limit_claims_per_channel = limit_claims_per_channel # doesn't count as censored
self.censored = {}
self.claims_in_channel = {}
self.total = 0
def censor(self, row) -> bool:
@@ -49,6 +51,11 @@ class Censor:
break
if was_censored:
self.total += 1
if not was_censored and self.limit_claims_per_channel is not None and row['channel_hash']:
self.claims_in_channel.setdefault(row['channel_hash'], 0)
self.claims_in_channel[row['channel_hash']] += 1
if self.claims_in_channel[row['channel_hash']] > self.limit_claims_per_channel:
return True
return was_censored
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
+2 -9
View File
@@ -22,8 +22,7 @@ def _create_url_regex():
return _group(
_named(name+"_name", prefix + invalid_names_regex) +
_oneof(
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
) + '?'
)
@@ -50,7 +49,6 @@ def normalize_name(name):
class PathSegment(NamedTuple):
name: str
claim_id: str = None
sequence: int = None
amount_order: int = None
@property
@@ -61,17 +59,13 @@ class PathSegment(NamedTuple):
q = {'name': self.name}
if self.claim_id is not None:
q['claim_id'] = self.claim_id
if self.sequence is not None:
q['sequence'] = self.sequence
if self.amount_order is not None:
q['amount_order'] = self.amount_order
return q
def __str__(self):
if self.claim_id is not None:
return f"{self.name}#{self.claim_id}"
elif self.sequence is not None:
return f"{self.name}:{self.sequence}"
return f"{self.name}:{self.claim_id}"
elif self.amount_order is not None:
return f"{self.name}${self.amount_order}"
return self.name
@@ -118,7 +112,6 @@ class URL(NamedTuple):
segments[segment] = PathSegment(
parts[f'{segment}_name'],
parts[f'{segment}_claim_id'],
parts[f'{segment}_sequence'],
parts[f'{segment}_amount_order']
)
+13 -3
View File
@@ -356,13 +356,23 @@ class ManagedStream(ManagedDownloadSource):
return sent
except ConnectionRefusedError:
return sent
except OSError:
# raised if a blob is deleted while it's being sent
return sent
except (OSError, Exception) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
raise err
if isinstance(err, OSError):
log.warning(
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
self.claim_id
)
else:
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
raise asyncio.CancelledError()
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+22 -6
View File
@@ -64,12 +64,12 @@ class StreamReflectorClient(asyncio.Protocol):
self.transport.write(msg.encode())
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
return await self.pending_request
except (AttributeError, asyncio.CancelledError):
except (AttributeError, asyncio.CancelledError) as err:
# attribute error happens when we transport.write after disconnect
# cancelled error happens when the pending_request task is cancelled by a disconnect
if self.transport:
self.transport.close()
raise asyncio.TimeoutError()
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
finally:
self.pending_request = None
@@ -94,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
needed = response.get('needed_blobs', [])
sent_sd = False
if response['send_sd_blob']:
await sd_blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
try:
sent = await sd_blob.sendfile(self)
if sent == -1:
log.warning("failed to send sd blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_sd_blob'):
sent_sd = True
if not needed:
@@ -118,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
if 'send_blob' not in response:
raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']:
await blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
try:
sent = await blob.sendfile(self)
if sent == -1:
log.warning("failed to send blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8])
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+29 -17
View File
@@ -7,8 +7,6 @@ from binascii import hexlify
from collections import defaultdict
from dataclasses import dataclass
from contextvars import ContextVar
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
from datetime import date
from prometheus_client import Gauge, Counter, Histogram
@@ -19,6 +17,12 @@ from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Inpu
from .constants import TXO_TYPES, CLAIM_TYPES
from .util import date_to_julian_day
from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' or 'KIVY_BUILD' in os.environ:
from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported
else:
from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass
log = logging.getLogger(__name__)
sqlite3.enable_callback_tracebacks(True)
@@ -62,12 +66,6 @@ def run_read_only_fetchone(sql, params):
raise
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' in os.environ:
ReaderExecutorClass = ThreadPoolExecutor
else:
ReaderExecutorClass = ProcessPoolExecutor
class AIOSQLite:
reader_executor: ReaderExecutorClass
@@ -123,7 +121,12 @@ class AIOSQLite:
if self._closing:
return
self._closing = True
await asyncio.get_event_loop().run_in_executor(self.writer_executor, self.writer_connection.close)
def __checkpoint_and_close(conn: sqlite3.Connection):
conn.execute("PRAGMA WAL_CHECKPOINT(FULL);")
conn.close()
await asyncio.get_event_loop().run_in_executor(
self.writer_executor, __checkpoint_and_close, self.writer_connection)
self.writer_executor.shutdown(wait=True)
self.reader_executor.shutdown(wait=True)
self.read_ready.clear()
@@ -147,7 +150,7 @@ class AIOSQLite:
self.waiting_reads_metric.inc()
self.read_count_metric.inc()
try:
while self.writers: # more writes can come in while we are waiting for the first
while self.writers and not self._closing: # more writes can come in while we are waiting for the first
if not urgent_read and still_waiting and self.urgent_read_done.is_set():
# throttle the writes if they pile up
self.urgent_read_done.clear()
@@ -155,6 +158,8 @@ class AIOSQLite:
# wait until the running writes have finished
await self.read_ready.wait()
still_waiting = True
if self._closing:
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.reader_executor, read_only_fn, sql, parameters
)
@@ -197,6 +202,8 @@ class AIOSQLite:
self.read_ready.clear()
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
)
@@ -232,6 +239,8 @@ class AIOSQLite:
self.read_ready.clear()
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
)
@@ -1254,13 +1263,16 @@ class Database(SQLiteMixin):
self.constrain_collections(constraints)
return self.get_utxo_count(**constraints)
async def release_all_outputs(self, account):
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
async def release_all_outputs(self, account=None):
if account is None:
await self.db.execute_fetchall("UPDATE txo SET is_reserved = 0 WHERE is_reserved = 1")
else:
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
def get_supports_summary(self, read_only=False, **constraints):
return self.get_txos(
+85 -53
View File
@@ -29,7 +29,6 @@ from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
from .bip32 import PubKey, PrivateKey
from .coinselection import CoinSelector
log = logging.getLogger(__name__)
LedgerType = Type['BaseLedger']
@@ -132,7 +131,7 @@ class Ledger(metaclass=LedgerRegistry):
self._on_transaction_controller = StreamController()
self.on_transaction = self._on_transaction_controller.stream
self.on_transaction.listen(
lambda e: log.info(
lambda e: log.debug(
'(%s) on_transaction: address=%s, height=%s, is_verified=%s, tx.id=%s',
self.get_id(), e.address, e.tx.height, e.tx.is_verified, e.tx.id
)
@@ -331,6 +330,7 @@ class Ledger(metaclass=LedgerRegistry):
async with self._header_processing_lock:
await self._update_tasks.add(self.initial_headers_sync())
await fully_synced
await self.db.release_all_outputs()
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
if len(self.accounts) > 10:
@@ -478,12 +478,14 @@ class Ledger(metaclass=LedgerRegistry):
for address, remote_status in zip(batch, results):
self._update_tasks.add(self.update_history(address, remote_status, address_manager))
addresses_remaining = addresses_remaining[batch_size:]
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
len(addresses), *self.network.client.server_address_and_port)
log.info(
"finished subscribing to %i addresses on %s:%i", len(addresses),
*self.network.client.server_address_and_port
)
if self.network.client and self.network.client.server_address_and_port:
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
len(addresses), *self.network.client.server_address_and_port)
if self.network.client and self.network.client.server_address_and_port:
log.info(
"finished subscribing to %i addresses on %s:%i", len(addresses),
*self.network.client.server_address_and_port
)
def process_status_update(self, update):
address, remote_status = update
@@ -635,40 +637,6 @@ class Ledger(metaclass=LedgerRegistry):
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
return True
async def cache_transaction(self, txid, remote_height, check_local=True):
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = self._tx_cache[txid] = TransactionCacheItem()
elif cache_item.tx is not None and \
cache_item.tx.height >= remote_height and \
(cache_item.tx.is_verified or remote_height < 1):
return cache_item.tx # cached tx is already up-to-date
cache_item.pending_verifications += 1
try:
async with cache_item.lock:
tx = cache_item.tx
if tx is None and check_local:
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
merkle = None
if tx is None:
# fetch from network
_raw, merkle = await self.network.retriable_call(
self.network.get_transaction_and_merkle, txid, remote_height
)
tx = Transaction(unhexlify(_raw), height=merkle['block_height'])
cache_item.tx = tx # make sure it's saved before caching it
tx.height = remote_height
if merkle and 0 < remote_height < len(self.headers):
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
return tx
finally:
cache_item.pending_verifications -= 1
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
tx.height = remote_height
cached = self._tx_cache.get(tx.id)
@@ -686,6 +654,56 @@ class Ledger(metaclass=LedgerRegistry):
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
async def request_transactions_for_inflate(self, to_request: Tuple[Tuple[str, int], ...]):
header_cache = {}
batches = [[]]
remote_heights = {}
transactions = []
heights_in_batch = 0
last_height = 0
for txid, height in sorted(to_request, key=lambda x: x[1]):
remote_heights[txid] = height
if height != last_height:
heights_in_batch += 1
last_height = height
if len(batches[-1]) == 100 or heights_in_batch == 20:
batches.append([])
heights_in_batch = 1
batches[-1].append(txid)
if not batches[-1]:
batches.pop()
async def _single_batch(batch):
batch_result = await self.network.retriable_call(
self.network.get_transaction_batch, batch, restricted=False
)
for txid, (raw, merkle) in batch_result.items():
remote_height = remote_heights[txid]
merkle_height = merkle['block_height']
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
tx = cache_item.tx or Transaction(bytes.fromhex(raw), height=remote_height)
tx.height = remote_height
cache_item.tx = tx
if 'merkle' in merkle and remote_heights[txid] > 0:
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
try:
header = header_cache.get(remote_heights[txid]) or (await self.headers.get(merkle_height))
except IndexError:
log.warning("failed to verify %s at height %i", tx.id, merkle_height)
else:
header_cache[remote_heights[txid]] = header
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
transactions.append(tx)
for batch in batches:
await _single_batch(batch)
return transactions
async def _request_transaction_batch(self, to_request, remote_history_size, address):
header_cache = {}
batches = [[]]
@@ -848,9 +866,8 @@ class Ledger(metaclass=LedgerRegistry):
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
txs = []
if len(outputs.txs) > 0:
txs: List[Transaction] = await asyncio.gather(*(
self.cache_transaction(*tx) for tx in outputs.txs
))
txs: List[Transaction] = []
txs.extend((await self.request_transactions_for_inflate(tuple(outputs.txs))))
_txos, blocked = outputs.inflate(txs)
@@ -922,13 +939,21 @@ class Ledger(metaclass=LedgerRegistry):
txo.received_tips = tips
return txos, blocked, outputs.offset, outputs.total
async def resolve(self, accounts, urls, **kwargs):
resolve = partial(self.network.retriable_call, self.network.resolve)
urls_copy = list(urls)
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
txos = []
urls_copy = list(urls)
if new_sdk_server:
resolve = partial(self.network.new_resolve, new_sdk_server)
else:
resolve = partial(self.network.retriable_call, self.network.resolve)
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0])
batch, urls_copy = urls_copy[:100], urls_copy[100:]
txos.extend(
(await self._inflate_outputs(
resolve(batch), accounts, **kwargs
))[0]
)
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
@@ -941,13 +966,20 @@ class Ledger(metaclass=LedgerRegistry):
result[url] = txo
return result
async def sum_supports(self, new_sdk_server, **kwargs) -> List[Dict]:
return await self.network.sum_supports(new_sdk_server, **kwargs)
async def claim_search(
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
**kwargs) -> Tuple[List[Output], dict, int, int]:
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
if new_sdk_server:
claim_search = partial(self.network.new_claim_search, new_sdk_server)
else:
claim_search = self.network.claim_search
return await self._inflate_outputs(
self.network.claim_search(**kwargs), accounts,
claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output
include_is_my_output=include_is_my_output,
)
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
+32 -8
View File
@@ -4,6 +4,7 @@ import json
from time import perf_counter
from operator import itemgetter
from typing import Dict, Optional, Tuple
import aiohttp
from lbry import __version__
from lbry.error import IncompatibleWalletServerError
@@ -181,6 +182,8 @@ class Network:
'blockchain.address.subscribe': self._on_status_controller,
}
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
@property
def config(self):
return self.ledger.config
@@ -207,6 +210,7 @@ class Network:
async def start(self):
self.running = True
self.aiohttp_session = aiohttp.ClientSession()
self._switch_task = asyncio.ensure_future(self.switch_forever())
# this may become unnecessary when there are no more bugs found,
# but for now it helps understanding log reports
@@ -217,6 +221,7 @@ class Network:
async def stop(self):
if self.running:
self.running = False
await self.aiohttp_session.close()
self._switch_task.cancel()
self.session_pool.stop()
@@ -224,8 +229,8 @@ class Network:
def is_connected(self):
return self.client and not self.client.is_closing()
def rpc(self, list_or_method, args, restricted=True):
session = self.client if restricted else self.session_pool.fastest_session
def rpc(self, list_or_method, args, restricted=True, session=None):
session = session or (self.client if restricted else self.session_pool.fastest_session)
if session and not session.is_closing():
return session.send_request(list_or_method, args)
else:
@@ -255,9 +260,9 @@ class Network:
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
def get_transaction_batch(self, txids):
def get_transaction_batch(self, txids, restricted=True, session=None):
# use any server if its old, otherwise restrict to who gave us the history
return self.rpc('blockchain.transaction.get_batch', txids, True)
return self.rpc('blockchain.transaction.get_batch', txids, restricted, session)
def get_transaction_and_merkle(self, tx_hash, known_height=None):
# use any server if its old, otherwise restrict to who gave us the history
@@ -310,11 +315,30 @@ class Network:
def get_claims_by_ids(self, claim_ids):
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def resolve(self, urls, session_override=None):
return self.rpc('blockchain.claimtrie.resolve', urls, False, session_override)
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
def claim_search(self, session_override=None, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override)
async def new_resolve(self, server, urls):
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
async def new_claim_search(self, server, **kwargs):
kwargs['protobuf'] = True
message = {"method": "claim_search", "params": kwargs}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
async def sum_supports(self, server, **kwargs):
message = {"method": "support_sum", "params": kwargs}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
class SessionPool:
+37 -36
View File
@@ -10,7 +10,7 @@ from lbry.wallet.server.db.writer import SQLDB
from lbry.wallet.server.daemon import DaemonError
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.util import chunks, class_logger
from lbry.wallet.server.leveldb import FlushData
from lbry.wallet.server.leveldb import FlushData, reopen_rocksdb_ctx, UTXO_PREFIX, HASHX_UTXO_PREFIX
class Prefetcher:
@@ -171,7 +171,8 @@ class BlockProcessor:
# Caches of unflushed items.
self.headers = []
self.tx_hashes = []
self.block_hashes = []
self.block_txs = []
self.undo_infos = []
# UTXO cache
@@ -274,6 +275,7 @@ class BlockProcessor:
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
await self.prefetcher.reset_height(self.height)
self.reorg_count_metric.inc()
await reopen_rocksdb_ctx(self.db.executor)
async def reorg_hashes(self, count):
"""Return a pair (start, last, hashes) of blocks to back up during a
@@ -288,7 +290,7 @@ class BlockProcessor:
self.logger.info(f'chain was reorganised replacing {count:,d} '
f'block{s} at heights {start:,d}-{last:,d}')
return start, last, await self.db.fs_block_hashes(start, count)
return start, last, self.db.fs_block_hashes(start, count)
async def calc_reorg_range(self, count: Optional[int]):
"""Calculate the reorg range"""
@@ -306,7 +308,7 @@ class BlockProcessor:
start = self.height - 1
count = 1
while start > 0:
hashes = await self.db.fs_block_hashes(start, count)
hashes = self.db.fs_block_hashes(start, count)
hex_hashes = [hash_to_hex_str(hash) for hash in hashes]
d_hex_hashes = await self.daemon.block_hex_hashes(start, count)
n = diff_pos(hex_hashes, d_hex_hashes)
@@ -336,8 +338,8 @@ class BlockProcessor:
def flush_data(self):
"""The data for a flush. The lock must be taken."""
assert self.state_lock.locked()
return FlushData(self.height, self.tx_count, self.headers,
self.tx_hashes, self.undo_infos, self.utxo_cache,
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
self.block_txs, self.undo_infos, self.utxo_cache,
self.db_deletes, self.tip)
async def flush(self, flush_utxos):
@@ -345,6 +347,7 @@ class BlockProcessor:
self.db.flush_dbs(self.flush_data(), flush_utxos,
self.estimate_txs_remaining)
await self.run_in_thread_with_lock(flush)
await reopen_rocksdb_ctx(self.db.executor)
async def _maybe_flush(self):
# If caught up, flush everything as client queries are
@@ -392,7 +395,8 @@ class BlockProcessor:
for block in blocks:
height += 1
undo_info = self.advance_txs(
height, block.transactions, self.coin.electrum_header(block.header, height)
height, block.transactions, self.coin.electrum_header(block.header, height),
self.coin.header_hash(block.header)
)
if height >= min_height:
self.undo_infos.append((undo_info, height))
@@ -403,25 +407,26 @@ class BlockProcessor:
self.headers.extend(headers)
self.tip = self.coin.header_hash(headers[-1])
def advance_txs(self, height, txs, header):
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
def advance_txs(self, height, txs, header, block_hash):
self.block_hashes.append(block_hash)
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
# Use local vars for speed in the loops
undo_info = []
tx_num = self.tx_count
script_hashX = self.coin.hashX_from_script
s_pack = pack
hashXs_by_tx = []
# Use local vars for speed in the loops
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
undo_info_append = undo_info.append
update_touched = self.touched.update
hashXs_by_tx = []
append_hashXs = hashXs_by_tx.append
append_hashX_by_tx = hashXs_by_tx.append
hashX_from_script = self.coin.hashX_from_script
for tx, tx_hash in txs:
hashXs = []
append_hashX = hashXs.append
tx_numb = s_pack('<I', tx_num)
tx_numb = pack('<I', tx_num)
# Spend the inputs
for txin in tx.inputs:
@@ -434,18 +439,17 @@ class BlockProcessor:
# Add the new UTXOs
for idx, txout in enumerate(tx.outputs):
# Get the hashX. Ignore unspendable outputs
hashX = script_hashX(txout.pk_script)
hashX = hashX_from_script(txout.pk_script)
if hashX:
append_hashX(hashX)
put_utxo(tx_hash + s_pack('<H', idx),
hashX + tx_numb + s_pack('<Q', txout.value))
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
append_hashXs(hashXs)
append_hashX_by_tx(hashXs)
update_touched(hashXs)
self.db.total_transactions.append(tx_hash)
tx_num += 1
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
self.tx_count = tx_num
self.db.tx_counts.append(tx_num)
@@ -487,20 +491,17 @@ class BlockProcessor:
# Use local vars for speed in the loops
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
script_hashX = self.coin.hashX_from_script
touched = self.touched
undo_entry_len = 12 + HASHX_LEN
for tx, tx_hash in reversed(txs):
self.db.total_transactions.pop()
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
hashX = script_hashX(txout.pk_script)
hashX = self.coin.hashX_from_script(txout.pk_script)
if hashX:
cache_value = spend_utxo(tx_hash, idx)
touched.add(cache_value[:-12])
cache_value = self.spend_utxo(tx_hash, idx)
self.touched.add(cache_value[:-12])
# Restore the inputs
for txin in reversed(tx.inputs):
@@ -508,9 +509,8 @@ class BlockProcessor:
continue
n -= undo_entry_len
undo_item = undo_info[n:n + undo_entry_len]
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
undo_item)
touched.add(undo_item[:-12])
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
self.touched.add(undo_item[:-12])
assert n == 0
self.tx_count -= len(txs)
@@ -586,9 +586,9 @@ class BlockProcessor:
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:4] + idx_packed
prefix = HASHX_UTXO_PREFIX + tx_hash[:4] + idx_packed
candidates = {db_key: hashX for db_key, hashX
in self.db.utxo_db.iterator(prefix=prefix)}
in self.db.db.iterator(prefix=prefix)}
for hdb_key, hashX in candidates.items():
tx_num_packed = hdb_key[-4:]
@@ -602,8 +602,8 @@ class BlockProcessor:
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
udb_key = b'u' + hashX + hdb_key[-6:]
utxo_value_packed = self.db.utxo_db.get(udb_key)
udb_key = UTXO_PREFIX + hashX + hdb_key[-6:]
utxo_value_packed = self.db.db.get(udb_key)
if utxo_value_packed:
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
@@ -768,10 +768,11 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
def advance_txs(self, height, txs, header):
def advance_txs(self, height, txs, header, block_hash):
timer = self.timer.sub_timers['advance_blocks']
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
self.timer.show(height=height)
+398 -200
View File
@@ -13,209 +13,407 @@ STREAM_TYPES = {
'model': 6
}
# 9/21/2020
MOST_USED_TAGS = {
"gaming",
"people & blogs",
"entertainment",
"music",
"pop culture",
"education",
"technology",
"blockchain",
"news",
"funny",
"science & technology",
"learning",
"gameplay",
"news & politics",
"comedy",
"bitcoin",
"beliefs",
"nature",
"art",
"economics",
"film & animation",
"lets play",
"games",
"sports",
"howto & style",
"game",
"cryptocurrency",
"playstation 4",
"automotive",
"crypto",
"mature",
"sony interactive entertainment",
"walkthrough",
"tutorial",
"video game",
"weapons",
"playthrough",
"pc",
"anime",
"how to",
"btc",
"fun",
"ethereum",
"food",
"travel & events",
"minecraft",
"science",
"autos & vehicles",
"play",
"politics",
"commentary",
"twitch",
"ps4live",
"love",
"ps4",
"nonprofits & activism",
"ps4share",
"fortnite",
"xbox",
"porn",
"video games",
"trump",
"español",
"money",
"music video",
"nintendo",
"movie",
"coronavirus",
"donald trump",
"steam",
"trailer",
"android",
"podcast",
"xbox one",
"survival",
"audio",
"linux",
"travel",
"funny moments",
"litecoin",
"animation",
"gamer",
"lets",
"playstation",
"bitcoin news",
"history",
"xxx",
"fox news",
"dance",
"god",
"adventure",
"liberal",
"2020",
"horror",
"government",
"freedom",
"reaction",
"meme",
"photography",
"truth",
"health",
"lbry",
"family",
"online",
"eth",
"crypto news",
"diy",
"trading",
"gold",
"memes",
"world",
"space",
"lol",
"covid-19",
"rpg",
"humor",
"democrat",
"film",
"call of duty",
"tech",
"religion",
"conspiracy",
"rap",
"cnn",
"hangoutsonair",
"unboxing",
"fiction",
"conservative",
"cars",
"hoa",
"epic",
"programming",
"progressive",
"cryptocurrency news",
"classical",
"jesus",
"movies",
"book",
"ps3",
"republican",
"fitness",
"books",
"multiplayer",
"animals",
"pokemon",
"bitcoin price",
"facebook",
"sharefactory",
"criptomonedas",
"cod",
"bible",
"business",
"stream",
"comics",
"how",
"fail",
"nsfw",
"new music",
"satire",
"pets & animals",
"computer",
"classical music",
"indie",
"musica",
"msnbc",
"fps",
"mod",
"sport",
"sony",
"ripple",
"auto",
"rock",
"marvel",
"complete",
"mining",
"political",
"mobile",
"pubg",
"hip hop",
"flat earth",
"xbox 360",
"reviews",
"vlogging",
"latest news",
"hack",
"tarot",
"iphone",
"media",
"cute",
"christian",
"free speech",
"trap",
"war",
"remix",
"ios",
"xrp",
"spirituality",
"song",
"league of legends",
"cat"
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
def normalize_tag(tag):
return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
}
INDEXED_LANGUAGES = [
'none',
'en',
'aa',
'ab',
'ae',
'af',
'ak',
'am',
'an',
'ar',
'as',
'av',
'ay',
'az',
'ba',
'be',
'bg',
'bh',
'bi',
'bm',
'bn',
'bo',
'br',
'bs',
'ca',
'ce',
'ch',
'co',
'cr',
'cs',
'cu',
'cv',
'cy',
'da',
'de',
'dv',
'dz',
'ee',
'el',
'eo',
'es',
'et',
'eu',
'fa',
'ff',
'fi',
'fj',
'fo',
'fr',
'fy',
'ga',
'gd',
'gl',
'gn',
'gu',
'gv',
'ha',
'he',
'hi',
'ho',
'hr',
'ht',
'hu',
'hy',
'hz',
'ia',
'id',
'ie',
'ig',
'ii',
'ik',
'io',
'is',
'it',
'iu',
'ja',
'jv',
'ka',
'kg',
'ki',
'kj',
'kk',
'kl',
'km',
'kn',
'ko',
'kr',
'ks',
'ku',
'kv',
'kw',
'ky',
'la',
'lb',
'lg',
'li',
'ln',
'lo',
'lt',
'lu',
'lv',
'mg',
'mh',
'mi',
'mk',
'ml',
'mn',
'mr',
'ms',
'mt',
'my',
'na',
'nb',
'nd',
'ne',
'ng',
'nl',
'nn',
'no',
'nr',
'nv',
'ny',
'oc',
'oj',
'om',
'or',
'os',
'pa',
'pi',
'pl',
'ps',
'pt',
'qu',
'rm',
'rn',
'ro',
'ru',
'rw',
'sa',
'sc',
'sd',
'se',
'sg',
'si',
'sk',
'sl',
'sm',
'sn',
'so',
'sq',
'sr',
'ss',
'st',
'su',
'sv',
'sw',
'ta',
'te',
'tg',
'th',
'ti',
'tk',
'tl',
'tn',
'to',
'tr',
'ts',
'tt',
'tw',
'ty',
'ug',
'uk',
'ur',
'uz',
've',
'vi',
'vo',
'wa',
'wo',
'xh',
'yi',
'yo',
'za',
'zh',
'zu'
]
+18 -5
View File
@@ -18,7 +18,7 @@ from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs, Censor
from lbry.wallet import Ledger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
from .full_text_search import FTS_ORDER_BY
@@ -39,7 +39,7 @@ ATTRIBUTE_ARRAY_MAX_LENGTH = 100
INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
'tx_position', 'channel_join', 'reposted',
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed',
'trending_local', 'trending_global',
@@ -96,8 +96,8 @@ class ReaderState:
def get_resolve_censor(self) -> Censor:
return Censor(self.blocked_streams, self.blocked_channels)
def get_search_censor(self) -> Censor:
return Censor(self.filtered_streams, self.filtered_channels)
def get_search_censor(self, limit_claims_per_channel: int) -> Censor:
return Censor(self.filtered_streams, self.filtered_channels, limit_claims_per_channel)
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
@@ -421,12 +421,13 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
assert set(constraints).issubset(SEARCH_PARAMS), \
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
total = None
limit_claims_per_channel = constraints.pop('limit_claims_per_channel', None)
if not constraints.pop('no_totals', False):
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
context = ctx.get()
search_censor = context.get_search_censor()
search_censor = context.get_search_censor(limit_claims_per_channel)
txo_rows = search_claims(search_censor, **constraints)
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
@@ -535,6 +536,18 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
AND tag IN ({values})
)
"""
elif attr == 'language':
indexed_languages = any_items & set(INDEXED_LANGUAGES)
if indexed_languages:
any_items -= indexed_languages
for language in indexed_languages:
any_queries[f'#_any_common_languages_{language}'] = f"""
EXISTS(
SELECT 1 FROM language INDEXED BY language_{language}_idx
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=language.claim_hash
AND language = '{language}'
)
"""
if any_items:
+323 -269
View File
@@ -1,18 +1,17 @@
"""
Delayed AR with variable decay rate.
The spike height function is also simpler.
AR-like trending with a delayed effect and a faster
decay rate for high valued claims.
"""
import copy
import math
import time
import apsw
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
HALF_LIFE = 200
# Whale threshold (higher -> less DB writing)
WHALE_THRESHOLD = 3.0
# Whale threshold, in LBC (higher -> less DB writing)
WHALE_THRESHOLD = 10000.0
# Decay coefficient per block
DECAY = 0.5**(1.0/HALF_LIFE)
@@ -38,6 +37,7 @@ def install(connection):
Install the trending algorithm.
"""
check_trending_values(connection)
trending_data.initialise(connection.cursor())
if TRENDING_LOG:
f = open("trending_variable_decay.log", "a")
@@ -46,7 +46,6 @@ def install(connection):
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
@@ -60,19 +59,304 @@ def check_trending_values(connection):
break
if needs_reset:
print("Resetting some columns. This might take a while...", flush=True, end="")
print("Resetting some columns. This might take a while...", flush=True,
end="")
c.execute(""" BEGIN;
UPDATE claim SET trending_group = 0;
UPDATE claim SET trending_mixed = 0;
UPDATE claim SET trending_global = 0;
UPDATE claim SET trending_local = 0;
COMMIT;""")
print("done.")
def spike_height(x, x_old):
def trending_log(s):
"""
Compute the size of a trending spike (normed - constant units).
Log a string to the log file
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
def trending_unit(height):
"""
Return the trending score unit at a given height.
"""
# Round to the beginning of a SAVE_INTERVAL batch of blocks.
_height = height - (height % SAVE_INTERVAL)
return 1.0/DECAY**(height % RENORM_INTERVAL)
class TrendingDB:
"""
An in-memory database of trending scores
"""
def __init__(self):
self.conn = apsw.Connection(":memory:")
self.cursor = self.conn.cursor()
self.initialised = False
self.write_needed = set()
def execute(self, query, *args, **kwargs):
return self.cursor.execute(query, *args, **kwargs)
def executemany(self, query, *args, **kwargs):
return self.cursor.executemany(query, *args, **kwargs)
def begin(self):
self.execute("BEGIN;")
def commit(self):
self.execute("COMMIT;")
def initialise(self, db):
"""
Pass in claims.db
"""
if self.initialised:
return
trending_log("Initialising trending database...")
# The need for speed
self.execute("PRAGMA JOURNAL_MODE=OFF;")
self.execute("PRAGMA SYNCHRONOUS=0;")
self.begin()
# Create the tables
self.execute("""
CREATE TABLE IF NOT EXISTS claims
(claim_hash BYTES PRIMARY KEY,
lbc REAL NOT NULL DEFAULT 0.0,
trending_score REAL NOT NULL DEFAULT 0.0)
WITHOUT ROWID;""")
self.execute("""
CREATE TABLE IF NOT EXISTS spikes
(id INTEGER PRIMARY KEY,
claim_hash BYTES NOT NULL,
height INTEGER NOT NULL,
mass REAL NOT NULL,
FOREIGN KEY (claim_hash)
REFERENCES claims (claim_hash));""")
# Clear out any existing data
self.execute("DELETE FROM claims;")
self.execute("DELETE FROM spikes;")
# Create indexes
self.execute("CREATE INDEX idx1 ON spikes (claim_hash, height, mass);")
self.execute("CREATE INDEX idx2 ON spikes (claim_hash, height, mass DESC);")
self.execute("CREATE INDEX idx3 on claims (lbc DESC, claim_hash, trending_score);")
# Import data from claims.db
for row in db.execute("""
SELECT claim_hash,
1E-8*(amount + support_amount) AS lbc,
trending_mixed
FROM claim;
"""):
self.execute("INSERT INTO claims VALUES (?, ?, ?);", row)
self.commit()
self.initialised = True
trending_log("done.\n")
def apply_spikes(self, height):
"""
Apply spikes that are due. This occurs inside a transaction.
"""
spikes = []
unit = trending_unit(height)
for row in self.execute("""
SELECT SUM(mass), claim_hash FROM spikes
WHERE height = ?
GROUP BY claim_hash;
""", (height, )):
spikes.append((row[0]*unit, row[1]))
self.write_needed.add(row[1])
self.executemany("""
UPDATE claims
SET trending_score = (trending_score + ?)
WHERE claim_hash = ?;
""", spikes)
self.execute("DELETE FROM spikes WHERE height = ?;", (height, ))
def decay_whales(self, height):
"""
Occurs inside transaction.
"""
if height % SAVE_INTERVAL != 0:
return
whales = self.execute("""
SELECT trending_score, lbc, claim_hash
FROM claims
WHERE lbc >= ?;
""", (WHALE_THRESHOLD, )).fetchall()
whales2 = []
for whale in whales:
trending, lbc, claim_hash = whale
# Overall multiplication factor for decay rate
# At WHALE_THRESHOLD, this is 1
# At 10*WHALE_THRESHOLD, it is 3
decay_rate_factor = 1.0 + 2.0*math.log10(lbc/WHALE_THRESHOLD)
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# Decay
trending *= factor
whales2.append((trending, claim_hash))
self.write_needed.add(claim_hash)
self.executemany("UPDATE claims SET trending_score=? WHERE claim_hash=?;",
whales2)
def renorm(self, height):
"""
Renormalise trending scores. Occurs inside a transaction.
"""
if height % RENORM_INTERVAL == 0:
threshold = 1.0E-3/DECAY_PER_RENORM
for row in self.execute("""SELECT claim_hash FROM claims
WHERE ABS(trending_score) >= ?;""",
(threshold, )):
self.write_needed.add(row[0])
self.execute("""UPDATE claims SET trending_score = ?*trending_score
WHERE ABS(trending_score) >= ?;""",
(DECAY_PER_RENORM, threshold))
def write_to_claims_db(self, db, height):
"""
Write changed trending scores to claims.db.
"""
if height % SAVE_INTERVAL != 0:
return
rows = self.execute(f"""
SELECT trending_score, claim_hash
FROM claims
WHERE claim_hash IN
({','.join('?' for _ in self.write_needed)});
""", self.write_needed).fetchall()
db.executemany("""UPDATE claim SET trending_mixed = ?
WHERE claim_hash = ?;""", rows)
# Clear list of claims needing to be written to claims.db
self.write_needed = set()
def update(self, db, height, recalculate_claim_hashes):
"""
Update trending scores.
Input is a cursor to claims.db, the block height, and the list of
claims that changed.
"""
assert self.initialised
self.begin()
self.renorm(height)
# Fetch changed/new claims from claims.db
for row in db.execute(f"""
SELECT claim_hash,
1E-8*(amount + support_amount) AS lbc
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
claim_hash, lbc = row
# Insert into trending db if it does not exist
self.execute("""
INSERT INTO claims (claim_hash)
VALUES (?)
ON CONFLICT (claim_hash) DO NOTHING;""",
(claim_hash, ))
# See if it was an LBC change
old = self.execute("SELECT * FROM claims WHERE claim_hash=?;",
(claim_hash, )).fetchone()
lbc_old = old[1]
# Save new LBC value into trending db
self.execute("UPDATE claims SET lbc = ? WHERE claim_hash = ?;",
(lbc, claim_hash))
if lbc > lbc_old:
# Schedule a future spike
delay = min(int((lbc + 1E-8)**0.4), HALF_LIFE)
spike = (claim_hash, height + delay, spike_mass(lbc, lbc_old))
self.execute("""INSERT INTO spikes
(claim_hash, height, mass)
VALUES (?, ?, ?);""", spike)
elif lbc < lbc_old:
# Subtract from future spikes
penalty = spike_mass(lbc_old, lbc)
spikes = self.execute("""
SELECT * FROM spikes
WHERE claim_hash = ?
ORDER BY height ASC, mass DESC;
""", (claim_hash, )).fetchall()
for spike in spikes:
spike_id, mass = spike[0], spike[3]
if mass > penalty:
# The entire penalty merely reduces this spike
self.execute("UPDATE spikes SET mass=? WHERE id=?;",
(mass - penalty, spike_id))
penalty = 0.0
else:
# Removing this spike entirely accounts for some (or
# all) of the penalty, then move on to other spikes
self.execute("DELETE FROM spikes WHERE id=?;",
(spike_id, ))
penalty -= mass
# If penalty remains, that's a negative spike to be applied
# immediately.
if penalty > 0.0:
self.execute("""
INSERT INTO spikes (claim_hash, height, mass)
VALUES (?, ?, ?);""",
(claim_hash, height, -penalty))
self.apply_spikes(height)
self.decay_whales(height)
self.commit()
self.write_to_claims_db(db, height)
# The "global" instance to work with
# pylint: disable=C0103
trending_data = TrendingDB()
def spike_mass(x, x_old):
"""
Compute the mass of a trending spike (normed - constant units).
x_old = old LBC value
x = new LBC value
"""
# Sign of trending spike
@@ -89,155 +373,16 @@ def spike_height(x, x_old):
return sign*mag
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log(f"Skipping trending calculations at block {height}.\n")
return
def get_time_boost(height):
"""
Return the time boost at a given height.
"""
return 1.0/DECAY**(height % RENORM_INTERVAL)
def trending_log(s):
"""
Log a string.
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
class TrendingData:
"""
An object of this class holds trending data
"""
def __init__(self):
# Dict from claim id to some trending info.
# Units are TIME VARIABLE in here
self.claims = {}
# Claims with >= WHALE_THRESHOLD LBC total amount
self.whales = set([])
# Have all claims been read from db yet?
self.initialised = False
# List of pending spikes.
# Units are CONSTANT in here
self.pending_spikes = []
def insert_claim_from_load(self, height, claim_hash, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
if trending_score >= WHALE_THRESHOLD*get_time_boost(height):
self.add_whale(claim_hash)
def add_whale(self, claim_hash):
self.whales.add(claim_hash)
def apply_spikes(self, height):
"""
Apply all pending spikes that are due at this height.
Apply with time boost ON.
"""
time_boost = get_time_boost(height)
for spike in self.pending_spikes:
if spike["height"] > height:
# Ignore
pass
if spike["height"] == height:
# Apply
self.claims[spike["claim_hash"]]["trending_score"] += time_boost*spike["size"]
self.claims[spike["claim_hash"]]["changed"] = True
if self.claims[spike["claim_hash"]]["trending_score"] >= WHALE_THRESHOLD*time_boost:
self.add_whale(spike["claim_hash"])
if spike["claim_hash"] in self.whales and \
self.claims[spike["claim_hash"]]["trending_score"] < WHALE_THRESHOLD*time_boost:
self.whales.remove(spike["claim_hash"])
# Keep only future spikes
self.pending_spikes = [s for s in self.pending_spikes \
if s["height"] > height]
def update_claim(self, height, claim_hash, total_amount):
"""
Update trending data for a claim, given its new total amount.
"""
assert self.initialised
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
"changed": False}
# Calculate LBC change
change = total_amount - old_state["total_amount"]
# Modify data if there was an LBC change
if change != 0.0:
spike = spike_height(total_amount,
old_state["total_amount"])
delay = min(int((total_amount + 1E-8)**0.4), HALF_LIFE)
if change < 0.0:
# How big would the spike be for the inverse movement?
reverse_spike = spike_height(old_state["total_amount"], total_amount)
# Remove that much spike from future pending ones
for future_spike in self.pending_spikes:
if future_spike["claim_hash"] == claim_hash:
if reverse_spike >= future_spike["size"]:
reverse_spike -= future_spike["size"]
future_spike["size"] = 0.0
elif reverse_spike > 0.0:
future_spike["size"] -= reverse_spike
reverse_spike = 0.0
delay = 0
spike = -reverse_spike
self.pending_spikes.append({"height": height + delay,
"claim_hash": claim_hash,
"size": spike})
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": old_state["trending_score"],
"changed": False}
def process_whales(self, height):
"""
Whale claims decay faster.
"""
if height % SAVE_INTERVAL != 0:
return
for claim_hash in self.whales:
trending_normed = self.claims[claim_hash]["trending_score"]/get_time_boost(height)
# Overall multiplication factor for decay rate
decay_rate_factor = trending_normed/WHALE_THRESHOLD
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# print(claim_hash, trending_normed, decay_rate_factor)
self.claims[claim_hash]["trending_score"] *= factor
self.claims[claim_hash]["changed"] = True
start = time.time()
trending_log(f"Calculating variable_decay trending at block {height}.\n")
trending_data.update(db, height, recalculate_claim_hashes)
end = time.time()
trending_log(f"Trending operations took {end - start} seconds.\n\n")
def test_trending():
"""
@@ -260,12 +405,12 @@ def test_trending():
COMMIT;
""")
# Initialise trending data before anything happens with the claims
trending_data.initialise(db)
# Insert initial states of claims
everything = {"huge_whale": 0.01,
"huge_whale_botted": 0.01,
"medium_whale": 0.01,
"small_whale": 0.01,
"minnow": 0.01}
everything = {"huge_whale": 0.01, "medium_whale": 0.01, "small_whale": 0.01,
"huge_whale_botted": 0.01, "minnow": 0.01}
def to_list_of_tuples(stuff):
l = []
@@ -277,13 +422,17 @@ def test_trending():
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
""", to_list_of_tuples(everything))
# Process block zero
height = 0
run(db, height, height, everything.keys())
# Save trajectories for plotting
trajectories = {}
for key in trending_data.claims:
trajectories[key] = [trending_data.claims[key]["trending_score"]]
for row in trending_data.execute("""
SELECT claim_hash, trending_score
FROM claims;
"""):
trajectories[row[0]] = [row[1]/trending_unit(height)]
# Main loop
for height in range(1, 1000):
@@ -312,119 +461,24 @@ def test_trending():
# Call run()
run(db, height, height, everything.keys())
for key in trending_data.claims:
trajectories[key].append(trending_data.claims[key]["trending_score"]\
/get_time_boost(height))
# Append current trending scores to trajectories
for row in db.execute("""
SELECT claim_hash, trending_mixed
FROM claim;
"""):
trajectories[row[0]].append(row[1]/trending_unit(height))
dbc.close()
# pylint: disable=C0415
import matplotlib.pyplot as plt
for key in trending_data.claims:
for key in trajectories:
plt.plot(trajectories[key], label=key)
plt.legend()
plt.show()
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping variable_decay trending at block {h}.\n".format(h=height))
return
start = time.time()
trending_log("Calculating variable_decay trending at block {h}.\n".format(h=height))
trending_log(" Length of trending data = {l}.\n"\
.format(l=len(trending_data.claims)))
# Renormalise trending scores and mark all as having changed
if height % RENORM_INTERVAL == 0:
trending_log(" Renormalising trending scores...")
keys = trending_data.claims.keys()
trending_data.whales = set([])
for key in keys:
if trending_data.claims[key]["trending_score"] != 0.0:
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
trending_data.claims[key]["changed"] = True
# Tiny becomes zero
if abs(trending_data.claims[key]["trending_score"]) < 1E-3:
trending_data.claims[key]["trending_score"] = 0.0
# Re-mark whales
if trending_data.claims[key]["trending_score"] >= WHALE_THRESHOLD*get_time_boost(height):
trending_data.add_whale(key)
trending_log("done.\n")
# Regular message.
trending_log(" Reading total_amounts from db and updating"\
+ " trending scores in RAM...")
# Update claims from db
if not trending_data.initialised:
trending_log("initial load...")
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
"""):
trending_data.insert_claim_from_load(height, row[0], row[1], 1E-8*row[2])
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
(amount + support_amount)
AS total_amount
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
trending_data.update_claim(height, row[0], 1E-8*row[1])
# Apply pending spikes
trending_data.apply_spikes(height)
trending_log("done.\n")
# Write trending scores to DB
if height % SAVE_INTERVAL == 0:
trending_log(" Finding and processing whales...")
trending_log(str(len(trending_data.whales)) + " whales found...")
trending_data.process_whales(height)
trending_log("done.\n")
trending_log(" Writing trending scores to db...")
the_list = []
keys = trending_data.claims.keys()
for key in keys:
if trending_data.claims[key]["changed"]:
the_list.append((trending_data.claims[key]["trending_score"], key))
trending_data.claims[key]["changed"] = False
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
the_list)
trending_log("done.\n")
trending_log("Trending operations took {time} seconds.\n\n"\
.format(time=time.time() - start))
if __name__ == "__main__":
+31 -4
View File
@@ -6,7 +6,6 @@ from decimal import Decimal
from collections import namedtuple
from multiprocessing import Manager
from binascii import unhexlify
from lbry.wallet.server.leveldb import LevelDB
from lbry.wallet.server.util import class_logger
from lbry.wallet.database import query, constraints_to_sql
@@ -19,7 +18,7 @@ from lbry.wallet.server.db.canonical import register_canonical_functions
from lbry.wallet.server.db.full_text_search import update_full_text_search, CREATE_FULL_TEXT_SEARCH, first_sync_finished
from lbry.wallet.server.db.trending import TRENDING_ALGORITHMS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -117,6 +116,15 @@ class SQLDB:
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
CREATE_LANGUAGE_TABLE = """
create table if not exists language (
language text not null,
claim_hash bytes not null,
height integer not null
);
create unique index if not exists language_claim_hash_language_idx on language (claim_hash, language);
"""
CREATE_CLAIMTRIE_TABLE = """
create table if not exists claimtrie (
normalized text primary key,
@@ -174,12 +182,18 @@ class SQLDB:
for tag_value, tag_key in COMMON_TAGS.items()
)
LANGUAGE_INDEXES = '\n'.join(
f"create unique index if not exists language_{language}_idx on language (language, claim_hash) WHERE language='{language}';"
for language in INDEXED_LANGUAGES
)
CREATE_TABLES_QUERY = (
CREATE_CLAIM_TABLE +
CREATE_FULL_TEXT_SEARCH +
CREATE_SUPPORT_TABLE +
CREATE_CLAIMTRIE_TABLE +
CREATE_TAG_TABLE
CREATE_TAG_TABLE +
CREATE_LANGUAGE_TABLE
)
def __init__(
@@ -305,7 +319,7 @@ class SQLDB:
self.execute('commit;')
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
claim_hashes, claims, tags = set(), [], {}
claim_hashes, claims, tags, languages = set(), [], {}, {}
for txo in txos:
tx = txo.tx_ref.tx
@@ -316,6 +330,13 @@ class SQLDB:
#self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.")
continue
language = 'none'
try:
if txo.claim.is_stream and txo.claim.stream.languages:
language = txo.claim.stream.languages[0].language
except:
pass
claim_hash = txo.claim_hash
claim_hashes.add(claim_hash)
claim_record = {
@@ -373,6 +394,8 @@ class SQLDB:
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
languages[(language, claim_hash)] = (language, claim_hash, tx.height)
for tag in clean_tags(claim.message.tags):
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
@@ -383,6 +406,10 @@ class SQLDB:
self.executemany(
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
)
if languages:
self.executemany(
"INSERT OR IGNORE INTO language (language, claim_hash, height) VALUES (?, ?, ?)", languages.values()
)
return claims
+4 -2
View File
@@ -36,8 +36,10 @@ class Env:
self.loop_policy = self.set_event_loop_policy()
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
self.db_engine = self.default('DB_ENGINE', 'rocksdb')
self.trending_algorithms = [
trending for trending in set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')) if trending
]
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
+40 -33
View File
@@ -20,6 +20,10 @@ from lbry.wallet.server.util import pack_be_uint16, unpack_be_uint16_from
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
HASHX_HISTORY_PREFIX = b'x'
HIST_STATE = b'state-hist'
class History:
DB_VERSIONS = [0]
@@ -32,8 +36,8 @@ class History:
self.unflushed_count = 0
self.db = None
def open_db(self, db_class, for_sync, utxo_flush_count, compacting):
self.db = db_class('hist', for_sync)
def open_db(self, db, for_sync, utxo_flush_count, compacting):
self.db = db #db_class('hist', for_sync)
self.read_state()
self.clear_excess(utxo_flush_count)
# An incomplete compaction needs to be cancelled otherwise
@@ -44,11 +48,11 @@ class History:
def close_db(self):
if self.db:
self.db.close()
# self.db.close()
self.db = None
def read_state(self):
state = self.db.get(b'state\0\0')
state = self.db.get(HIST_STATE)
if state:
state = ast.literal_eval(state.decode())
if not isinstance(state, dict):
@@ -80,17 +84,18 @@ class History:
'excess history flushes...')
keys = []
for key, hist in self.db.iterator(prefix=b''):
flush_id, = unpack_be_uint16_from(key[-2:])
for key, hist in self.db.iterator(prefix=HASHX_HISTORY_PREFIX):
k = key[1:]
flush_id, = unpack_be_uint16_from(k[-2:])
if flush_id > utxo_flush_count:
keys.append(key)
keys.append(k)
self.logger.info(f'deleting {len(keys):,d} history entries')
self.flush_count = utxo_flush_count
with self.db.write_batch() as batch:
for key in keys:
batch.delete(key)
batch.delete(HASHX_HISTORY_PREFIX + key)
self.write_state(batch)
self.logger.info('deleted excess history entries')
@@ -105,7 +110,7 @@ class History:
}
# History entries are not prefixed; the suffix \0\0 ensures we
# look similar to other entries and aren't interfered with
batch.put(b'state\0\0', repr(state).encode())
batch.put(HIST_STATE, repr(state).encode())
def add_unflushed(self, hashXs_by_tx, first_tx_num):
unflushed = self.unflushed
@@ -132,7 +137,7 @@ class History:
with self.db.write_batch() as batch:
for hashX in sorted(unflushed):
key = hashX + flush_id
batch.put(key, unflushed[hashX].tobytes())
batch.put(HASHX_HISTORY_PREFIX + key, unflushed[hashX].tobytes())
self.write_state(batch)
count = len(unflushed)
@@ -154,16 +159,17 @@ class History:
for hashX in sorted(hashXs):
deletes = []
puts = {}
for key, hist in self.db.iterator(prefix=hashX, reverse=True):
for key, hist in self.db.iterator(prefix=HASHX_HISTORY_PREFIX + hashX, reverse=True):
k = key[1:]
a = array.array('I')
a.frombytes(hist)
# Remove all history entries >= tx_count
idx = bisect_left(a, tx_count)
nremoves += len(a) - idx
if idx > 0:
puts[key] = a[:idx].tobytes()
puts[k] = a[:idx].tobytes()
break
deletes.append(key)
deletes.append(k)
for key in deletes:
batch.delete(key)
@@ -173,20 +179,20 @@ class History:
self.logger.info(f'backing up removed {nremoves:,d} history entries')
def get_txnums(self, hashX, limit=1000):
"""Generator that returns an unpruned, sorted list of tx_nums in the
history of a hashX. Includes both spending and receiving
transactions. By default yields at most 1000 entries. Set
limit to None to get them all. """
limit = util.resolve_limit(limit)
for key, hist in self.db.iterator(prefix=hashX):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
if limit == 0:
return
yield tx_num
limit -= 1
# def get_txnums(self, hashX, limit=1000):
# """Generator that returns an unpruned, sorted list of tx_nums in the
# history of a hashX. Includes both spending and receiving
# transactions. By default yields at most 1000 entries. Set
# limit to None to get them all. """
# limit = util.resolve_limit(limit)
# for key, hist in self.db.iterator(prefix=hashX):
# a = array.array('I')
# a.frombytes(hist)
# for tx_num in a:
# if limit == 0:
# return
# yield tx_num
# limit -= 1
#
# History compaction
@@ -221,9 +227,9 @@ class History:
with self.db.write_batch() as batch:
# Important: delete first! The keyspace may overlap.
for key in keys_to_delete:
batch.delete(key)
batch.delete(HASHX_HISTORY_PREFIX + key)
for key, value in write_items:
batch.put(key, value)
batch.put(HASHX_HISTORY_PREFIX + key, value)
self.write_state(batch)
def _compact_hashX(self, hashX, hist_map, hist_list,
@@ -271,11 +277,12 @@ class History:
key_len = HASHX_LEN + 2
write_size = 0
for key, hist in self.db.iterator(prefix=prefix):
for key, hist in self.db.iterator(prefix=HASHX_HISTORY_PREFIX + prefix):
k = key[1:]
# Ignore non-history entries
if len(key) != key_len:
if len(k) != key_len:
continue
hashX = key[:-2]
hashX = k[:-2]
if hashX != prior_hashX and prior_hashX:
write_size += self._compact_hashX(prior_hashX, hist_map,
hist_list, write_items,
@@ -283,7 +290,7 @@ class History:
hist_map.clear()
hist_list.clear()
prior_hashX = hashX
hist_map[key] = hist
hist_map[k] = hist
hist_list.append(hist)
if prior_hashX:
+433 -211
View File
@@ -14,27 +14,330 @@ import array
import ast
import os
import time
import zlib
import pylru
import typing
from typing import Optional, List, Tuple, Iterable
from asyncio import sleep
from bisect import bisect_right
from collections import namedtuple
from functools import partial
from glob import glob
from struct import pack, unpack
from contextvars import ContextVar
from dataclasses import dataclass
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
import attr
from lbry.wallet.server import util
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.merkle import Merkle, MerkleCache
from lbry.wallet.server.util import formatted_time
from lbry.wallet.server.storage import db_class
from lbry.wallet.server.util import formatted_time, unpack_be_uint64, unpack_le_int32_from, pack_le_int32
from lbry.wallet.server.storage import db_class, RocksDB
from lbry.wallet.server.history import History
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
HEADER_PREFIX = b'H'
TX_COUNT_PREFIX = b'T'
TX_HASH_PREFIX = b'X'
TX_PREFIX = b'B'
TX_NUM_PREFIX = b'N'
BLOCK_HASH_PREFIX = b'C'
HISTORY_PREFIX = b'A'
HASHX_UTXO_PREFIX = b'h'
UTXO_PREFIX = b'u'
HASHX_HISTORY_PREFIX = b'x'
UNDO_PREFIX = b'U'
UTXO_STATE = b'state-utxo'
HIST_STATE = b'state-hist'
TX_COUNTS_STATE = b'state-tx-counts'
@dataclass
class RocksReaderContext:
db: 'RocksDB'
db_dir: str
name: str
merkle: Merkle
tx_counts: List[int]
block_txs_cache: pylru.lrucache
merkle_tx_cache: pylru.lrucache
txid_cache: pylru.lrucache
def close(self):
self.db.close()
def reopen(self):
self.db.close()
self.db.open(self.name, create=False, read_only=True)
def update_state(self):
tx_counts = array.array("L")
counts = self.db.get(TX_COUNTS_STATE)
if not counts:
self.tx_counts = list(map(unpack_be_uint64, self.db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)))
else:
tx_counts.frombytes(counts)
self.tx_counts = tx_counts.tolist()
def ctx_tx_hash(self, tx_num):
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > len(self.tx_counts):
return None, tx_height
key = TX_HASH_PREFIX + util.pack_be_uint64(tx_num)
if key in self.txid_cache:
return self.txid_cache[key], tx_height
tx_hash = self.db.get(key)
if tx_height + 100 <= len(self.tx_counts):
self.txid_cache[key] = tx_hash
return tx_hash, tx_height
proc_ctx: ContextVar[Optional[RocksReaderContext]] = ContextVar('proc_ctx')
def _update_rocksdb_ctx():
ctx = proc_ctx.get()
ctx.update_state()
def _reopen_rocksdb_ctx():
ctx = proc_ctx.get()
ctx.db.close()
ctx.db = RocksDB(ctx.db_dir, ctx.name, for_sync=False, read_only=True)
ctx.update_state()
async def update_rocksdb_ctx(executor: ProcessPoolExecutor):
loop = asyncio.get_event_loop()
await asyncio.wait([loop.run_in_executor(executor, _update_rocksdb_ctx) for _ in range(executor._max_workers)])
async def reopen_rocksdb_ctx(executor: ProcessPoolExecutor):
loop = asyncio.get_event_loop()
await asyncio.wait([loop.run_in_executor(executor, _reopen_rocksdb_ctx) for _ in range(executor._max_workers)])
def _initializer(path, name):
db = RocksDB(path, name, for_sync=False, read_only=True)
proc_ctx.set(RocksReaderContext(db, path, name, Merkle(), [], pylru.lrucache(50000), pylru.lrucache(100000),
pylru.lrucache(1000000)))
def _teardown():
proc_ctx.get().close()
proc_ctx.set(None)
async def initialize_executor(workers, db_dir, for_sync, name):
executor = ProcessPoolExecutor(workers, initializer=_initializer, initargs=(db_dir, name))
try:
writer = RocksDB(db_dir, name, for_sync=for_sync, read_only=False)
await update_rocksdb_ctx(executor)
except Exception as err:
await teardown_executor(executor)
executor.shutdown(True)
raise err
return executor, writer
async def teardown_executor(executor: ProcessPoolExecutor):
try:
await asyncio.wait(
[asyncio.get_event_loop().run_in_executor(executor, _teardown) for _ in range(executor._max_workers)]
)
finally:
executor.shutdown(True)
def lookup_hashXs_utxos(prevouts):
"""Return (hashX, suffix) pairs, or None if not found,
for each prevout.
"""
ctx = proc_ctx.get()
iterator = ctx.db.iterator
get = ctx.db.get
def lookup_hashXs():
"""Return (hashX, suffix) pairs, or None if not found,
for each prevout.
"""
def lookup_hashX(tx_hash, tx_idx):
idx_packed = pack('<H', tx_idx)
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = HASHX_UTXO_PREFIX + tx_hash[:4] + idx_packed
# Find which entry, if any, the TX_HASH matches.
for db_key, hashX in iterator(prefix=prefix):
tx_num_packed = db_key[-4:]
tx_num, = unpack('<I', tx_num_packed)
hash, height = ctx.ctx_tx_hash(tx_num)
if hash == tx_hash:
return hashX, idx_packed + tx_num_packed
return None, None
return [lookup_hashX(*prevout) for prevout in prevouts]
def lookup_utxo(hashX, suffix):
if not hashX:
# This can happen when the daemon is a block ahead
# of us and has mempool txs spending outputs from
# that new block
return None
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
db_value = get(UTXO_PREFIX + hashX + suffix)
if not db_value:
# This can happen if the DB was updated between
# getting the hashXs and getting the UTXOs
return None
value, = unpack('<Q', db_value)
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in lookup_hashXs()]
def get_counts():
ctx = proc_ctx.get()
return ctx.tx_counts
def read_txids():
return list(proc_ctx.get().db.iterator(prefix=TX_HASH_PREFIX, include_key=False))
def read_headers():
return [
header for header in proc_ctx.get().db.iterator(prefix=HEADER_PREFIX, include_key=False)
]
def read_block_file(path):
with util.open_file(path) as f:
return f.read(-1)
def tx_hash(self, tx_num):
"""Return a par (tx_hash, tx_height) for the given tx number.
If the tx_height is not on disk, returns (None, tx_height)."""
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
return None, tx_height
return self.total_transactions[tx_num], tx_height
def read_utxos(hashX):
utxos = []
utxos_append = utxos.append
s_unpack = unpack
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
for db_key, db_value in proc_ctx.get().db.iterator(prefix=UTXO_PREFIX + hashX):
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
value, = unpack('<Q', db_value)
utxos_append((tx_num, tx_pos, value))
return utxos
def limited_history(hashX, limit=None):
ctx = proc_ctx.get()
tx_counts = ctx.tx_counts
db_height = len(tx_counts)
cnt = 0
txs = []
for hist in ctx.db.iterator(prefix=HASHX_HISTORY_PREFIX + hashX, include_key=False):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
tx_height = bisect_right(tx_counts, tx_num)
if tx_height > db_height:
return
txs.append((tx_num, tx_height))
cnt += 1
if limit and cnt >= limit:
break
if limit and cnt >= limit:
break
return txs
def tx_merkle(tx_num, tx_height):
ctx = proc_ctx.get()
db = ctx.db
tx_counts = ctx.tx_counts
db_height = len(tx_counts)
if tx_height == -1:
return {
'block_height': -1
}
tx_pos = tx_num - tx_counts[tx_height - 1]
uncached = None
if (tx_num, tx_height) in ctx.merkle_tx_cache:
return ctx.merkle_tx_cache[(tx_num, tx_height)]
if tx_height not in ctx.block_txs_cache:
block_txs = list(db.iterator(
prefix=TX_HASH_PREFIX,
start=TX_HASH_PREFIX + util.pack_be_uint64(tx_counts[tx_height - 1]),
stop=None if tx_height + 1 == len(tx_counts) else
TX_HASH_PREFIX + util.pack_be_uint64(tx_counts[tx_height] - 1), include_key=False
))
# print("rocks txs", tx_height, list(map(hash_to_hex_str, block_txs)))
if tx_height + 100 <= db_height:
ctx.block_txs_cache[tx_height] = block_txs
else:
block_txs = ctx.block_txs_cache.get(tx_height, uncached)
branch, root = ctx.merkle.branch_and_root(block_txs, tx_pos)
merkle = {
'block_height': tx_height,
'merkle': [
hash_to_hex_str(hash)
for hash in branch
],
'pos': tx_pos
}
if tx_height + 100 < db_height:
ctx.merkle_tx_cache[(tx_num, tx_height)] = merkle
return merkle
def transaction_info_get_batch(txids: Iterable[str]):
ctx = proc_ctx.get()
db_height = len(ctx.tx_counts)
tx_counts = ctx.tx_counts
tx_db_get = ctx.db.get
tx_infos = {}
for tx_hash in txids:
tx_hash_bytes = bytes.fromhex(tx_hash)[::-1]
tx_num = tx_db_get(TX_NUM_PREFIX + tx_hash_bytes)
tx = None
tx_height = -1
if tx_num is not None:
tx_num = unpack_be_uint64(tx_num)
tx_height = bisect_right(tx_counts, tx_num)
if tx_height < db_height:
tx = tx_db_get(TX_PREFIX + tx_hash_bytes)
tx_infos[tx_hash] = (
None if not tx else tx.hex(), {'block_height': -1} if tx_height == -1 else tx_merkle(
tx_num, tx_height
)
)
return tx_infos
def _update_block_txs_cache(tx_num, tx_height):
ctx = proc_ctx.get()
db = ctx.db
tx_counts = ctx.tx_counts
db_height = len(tx_counts)
@attr.s(slots=True)
@@ -42,7 +345,8 @@ class FlushData:
height = attr.ib()
tx_count = attr.ib()
headers = attr.ib()
block_tx_hashes = attr.ib()
block_hashes = attr.ib()
block_txs = attr.ib()
# The following are flushed to the UTXO DB if undo_infos is not None
undo_infos = attr.ib()
adds = attr.ib()
@@ -72,8 +376,9 @@ class LevelDB:
self.db_class = db_class(env.db_dir, self.env.db_engine)
self.history = History()
self.utxo_db = None
self.db = None
self.tx_counts = None
self.headers = None
self.last_flush = time.time()
self.logger.info(f'using {self.env.db_engine} for DB backend')
@@ -82,9 +387,9 @@ class LevelDB:
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
self.hashes_db = None
self.headers_db = None
self.tx_count_db = None
self._block_txs_cache = pylru.lrucache(50000)
self._merkle_tx_cache = pylru.lrucache(100000)
self.total_transactions = None
async def _read_tx_counts(self):
if self.tx_counts is not None:
@@ -92,15 +397,9 @@ class LevelDB:
# tx_counts[N] has the cumulative number of txs at the end of
# height N. So tx_counts[0] is 1 - the genesis coinbase
def get_counts():
return tuple(
util.unpack_be_uint64(tx_count)
for tx_count in self.tx_count_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
)
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
assert len(tx_counts) == self.db_height + 1, f"{len(tx_counts)} vs {self.db_height + 1}"
self.tx_counts = array.array('I', tx_counts)
self.tx_counts = tx_counts
if self.tx_counts:
assert self.db_tx_count == self.tx_counts[-1], \
@@ -108,58 +407,54 @@ class LevelDB:
else:
assert self.db_tx_count == 0
async def _read_txids(self):
start = time.perf_counter()
self.logger.info("loading txids")
txids = await asyncio.get_event_loop().run_in_executor(self.executor, read_txids)
assert len(txids) == len(self.tx_counts) == 0 or len(txids) == self.tx_counts[-1]
self.total_transactions = txids
ts = time.perf_counter() - start
self.logger.info("loaded %i txids in %ss", len(self.total_transactions), round(ts, 4))
async def _read_headers(self):
if self.headers is not None:
return
headers = await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}"
self.headers = headers
async def _open_dbs(self, for_sync, compacting):
name = f'lbry-{self.env.db_engine}'
if self.executor is None:
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
coin_path = os.path.join(self.env.db_dir, 'COIN')
if not os.path.isfile(coin_path):
with util.open_file(coin_path, create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
self.executor, self.db = await initialize_executor(
max(1, os.cpu_count() - 1), self.env.db_dir, for_sync, name
)
assert self.headers_db is None
self.headers_db = self.db_class('headers', for_sync)
if self.headers_db.is_new:
self.logger.info('created new headers db')
self.logger.info(f'opened headers DB (for sync: {for_sync})')
if self.db.is_new:
self.logger.info('created new db: %s', name)
self.logger.info(f'opened DB (for sync: {for_sync})')
assert self.tx_count_db is None
self.tx_count_db = self.db_class('tx_count', for_sync)
if self.tx_count_db.is_new:
self.logger.info('created new tx count db')
self.logger.info(f'opened tx count DB (for sync: {for_sync})')
assert self.hashes_db is None
self.hashes_db = self.db_class('hashes', for_sync)
if self.hashes_db.is_new:
self.logger.info('created new tx hashes db')
self.logger.info(f'opened tx hashes DB (for sync: {for_sync})')
assert self.utxo_db is None
# First UTXO DB
self.utxo_db = self.db_class('utxo', for_sync)
if self.utxo_db.is_new:
self.logger.info('created new utxo db')
self.logger.info(f'opened utxo db (for sync: {for_sync})')
self.read_utxo_state()
# Then history DB
self.utxo_flush_count = self.history.open_db(
self.db_class, for_sync, self.utxo_flush_count, compacting
self.db, for_sync, self.utxo_flush_count, compacting
)
self.clear_excess_undo_info()
# Read TX counts (requires meta directory)
await self._read_tx_counts()
if self.total_transactions is None:
await self._read_txids()
await self._read_headers()
def close(self):
self.utxo_db.close()
self.db.close()
self.history.close_db()
self.headers_db.close()
self.tx_count_db.close()
self.hashes_db.close()
self.executor.shutdown(wait=True)
self.executor = None
self.db = None
async def open_for_compacting(self):
await self._open_dbs(True, True)
@@ -178,21 +473,12 @@ class LevelDB:
"""Open the databases for serving. If they are already open they are
closed first.
"""
self.logger.info('closing DBs to re-open for serving')
if self.utxo_db:
self.logger.info('closing DBs to re-open for serving')
self.utxo_db.close()
self.history.close_db()
self.utxo_db = None
if self.headers_db:
self.headers_db.close()
self.headers_db = None
if self.tx_count_db:
self.tx_count_db.close()
self.tx_count_db = None
if self.hashes_db:
self.hashes_db.close()
self.hashes_db = None
if self.db:
return
# self.logger.info('closing DBs to re-open for serving')
# self.db.close()
# self.history.close_db()
# self.db = None
await self._open_dbs(False, False)
self.logger.info("opened for serving")
@@ -217,7 +503,7 @@ class LevelDB:
assert flush_data.height == self.fs_height == self.db_height
assert flush_data.tip == self.db_tip
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert not flush_data.block_txs
assert not flush_data.adds
assert not flush_data.deletes
assert not flush_data.undo_infos
@@ -241,14 +527,14 @@ class LevelDB:
self.flush_history()
# Flush state last as it reads the wall time.
with self.utxo_db.write_batch() as batch:
with self.db.write_batch() as batch:
if flush_utxos:
self.flush_utxo_db(batch, flush_data)
self.flush_state(batch)
# Update and put the wall time again - otherwise we drop the
# time it took to commit the batch
self.flush_state(self.utxo_db)
self.flush_state(self.db)
elapsed = self.last_flush - start_time
self.logger.info(f'flush #{self.history.flush_count:,d} took '
@@ -256,7 +542,7 @@ class LevelDB:
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
# Catch-up stats
if self.utxo_db.for_sync:
if self.db.for_sync:
flush_interval = self.last_flush - prior_flush
tx_per_sec_gen = int(flush_data.tx_count / self.wall_time)
tx_per_sec_last = 1 + int(tx_delta / flush_interval)
@@ -275,36 +561,57 @@ class LevelDB:
"""
prior_tx_count = (self.tx_counts[self.fs_height]
if self.fs_height >= 0 else 0)
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
assert len(flush_data.block_txs) == len(flush_data.headers)
assert flush_data.height == self.fs_height + len(flush_data.headers)
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
else 0)
assert len(self.tx_counts) == flush_data.height + 1
assert len(b''.join(flush_data.block_tx_hashes)) // 32 == flush_data.tx_count - prior_tx_count
assert len(
b''.join(hashes for hashes, _ in flush_data.block_txs)
) // 32 == flush_data.tx_count - prior_tx_count
# Write the headers, tx counts, and tx hashes
# Write the headers
start_time = time.perf_counter()
height_start = self.fs_height + 1
tx_num = prior_tx_count
for header, tx_hashes in zip(flush_data.headers, flush_data.block_tx_hashes):
tx_count = self.tx_counts[height_start]
self.headers_db.put(HEADER_PREFIX + util.pack_be_uint64(height_start), header)
self.tx_count_db.put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
self.hashes_db.put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
tx_num += 1
offset += 32
with self.db.write_batch() as batch:
batch_put = batch.put
for i, header in enumerate(flush_data.headers):
batch_put(HEADER_PREFIX + util.pack_be_uint64(self.fs_height + i + 1), header)
self.headers.append(header)
flush_data.headers.clear()
height_start = self.fs_height + 1
tx_num = prior_tx_count
for block_hash, (tx_hashes, txs) in zip(flush_data.block_hashes, flush_data.block_txs):
tx_count = self.tx_counts[height_start]
batch_put(BLOCK_HASH_PREFIX + util.pack_be_uint64(height_start), block_hash[::-1])
batch_put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
batch_put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
batch_put(TX_NUM_PREFIX + tx_hashes[offset:offset+32], util.pack_be_uint64(tx_num))
batch_put(TX_PREFIX + tx_hashes[offset:offset+32], txs[offset // 32])
tx_num += 1
offset += 32
a = array.array('L')
a.fromlist(self.tx_counts)
batch_put(TX_COUNTS_STATE, a.tobytes())
flush_data.block_txs.clear()
flush_data.block_hashes.clear()
flush_data.block_tx_hashes.clear()
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
flush_data.headers.clear()
elapsed = time.perf_counter() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
def flush_history(self):
self.history.flush()
@@ -329,15 +636,15 @@ class LevelDB:
# suffix = tx_idx + tx_num
hashX = value[:-12]
suffix = key[-2:] + value[-12:-8]
batch_put(b'h' + key[:4] + suffix, hashX)
batch_put(b'u' + hashX + suffix, value[-8:])
batch_put(HASHX_UTXO_PREFIX + key[:4] + suffix, hashX)
batch_put(UTXO_PREFIX + hashX + suffix, value[-8:])
flush_data.adds.clear()
# New undo information
self.flush_undo_infos(batch_put, flush_data.undo_infos)
flush_data.undo_infos.clear()
if self.utxo_db.for_sync:
if self.db.for_sync:
block_count = flush_data.height - self.db_height
tx_count = flush_data.tx_count - self.db_tx_count
elapsed = time.time() - start_time
@@ -362,7 +669,7 @@ class LevelDB:
def flush_backup(self, flush_data, touched):
"""Like flush_dbs() but when backing up. All UTXOs are flushed."""
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert not flush_data.block_txs
assert flush_data.height < self.db_height
self.history.assert_flushed()
@@ -371,7 +678,7 @@ class LevelDB:
self.backup_fs(flush_data.height, flush_data.tx_count)
self.history.backup(touched, flush_data.tx_count)
with self.utxo_db.write_batch() as batch:
with self.db.write_batch() as batch:
self.flush_utxo_db(batch, flush_data)
# Flush state last as it reads the wall time.
self.flush_state(batch)
@@ -383,19 +690,21 @@ class LevelDB:
def backup_fs(self, height, tx_count):
"""Back up during a reorg. This just updates our pointers."""
self.fs_height = height
while self.fs_height > height:
self.fs_height -= 1
self.headers.pop()
self.fs_tx_count = tx_count
# Truncate header_mc: header count is 1 more than the height.
self.header_mc.truncate(height + 1)
async def raw_header(self, height):
def raw_header(self, height):
"""Return the binary header at the given height."""
header, n = await self.read_headers(height, 1)
header, n = self.read_headers(height, 1)
if n != 1:
raise IndexError(f'height {height:,d} out of range')
return header
async def read_headers(self, start_height, count):
def read_headers(self, start_height, count) -> typing.Tuple[bytes, int]:
"""Requires start_height >= 0, count >= 0. Reads as many headers as
are available starting at start_height up to count. This
would be zero if start_height is beyond self.db_height, for
@@ -404,24 +713,15 @@ class LevelDB:
Returns a (binary, n) pair where binary is the concatenated
binary headers, and n is the count of headers returned.
"""
if start_height < 0 or count < 0:
raise self.DBError(f'{count:,d} headers starting at '
f'{start_height:,d} not on disk')
def read_headers():
# Read some from disk
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(
self.headers_db.iterator(
start=HEADER_PREFIX + util.pack_be_uint64(start_height),
stop=HEADER_PREFIX + util.pack_be_uint64(start_height + disk_count),
include_key=False
)
), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(self.headers[start_height:start_height + disk_count]), disk_count
return b'', 0
def fs_tx_hash(self, tx_num):
"""Return a par (tx_hash, tx_height) for the given tx number.
@@ -429,22 +729,18 @@ class LevelDB:
If the tx_height is not on disk, returns (None, tx_height)."""
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
return tx_hash, tx_height
return None, tx_height
return self.total_transactions[tx_num], tx_height
async def fs_block_hashes(self, height, count):
headers_concat, headers_count = await self.read_headers(height, count)
if headers_count != count:
raise self.DBError(f'only got {headers_count:,d} headers starting at {height:,d}, not {count:,d}')
offset = 0
headers = []
for n in range(count):
headers.append(headers_concat[offset:offset + self.coin.BASIC_HEADER_SIZE])
offset += self.coin.BASIC_HEADER_SIZE
async def fs_transactions(self, txids):
return await asyncio.get_event_loop().run_in_executor(
self.executor, transaction_info_get_batch, txids
)
return [self.coin.header_hash(header) for header in headers]
def fs_block_hashes(self, height, count):
if height + count > len(self.headers):
raise self.DBError(f'only got {len(self.headers) - height:,d} headers starting at {height:,d}, not {count:,d}')
return [self.coin.header_hash(header) for header in self.headers[height:height + count]]
async def limited_history(self, hashX, *, limit=1000):
"""Return an unpruned, sorted list of (tx_hash, height) tuples of
@@ -453,26 +749,11 @@ class LevelDB:
transactions. By default returns at most 1000 entries. Set
limit to None to get them all.
"""
def read_history():
hashx_history = []
for key, hist in self.history.db.iterator(prefix=hashX):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
hashx_history.append((tx_hash, tx_height))
if limit and len(hashx_history) >= limit:
return hashx_history
return hashx_history
while True:
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
if all(hash is not None for hash, height in history):
return history
history = await asyncio.get_event_loop().run_in_executor(self.executor, limited_history, hashX, limit)
if history is not None:
return [(self.total_transactions[tx_num], tx_height) for (tx_num, tx_height) in history]
self.logger.warning(f'limited_history: tx hash '
f'not found (reorg?), retrying...')
await sleep(0.25)
@@ -485,11 +766,11 @@ class LevelDB:
def undo_key(self, height):
"""DB key for undo information at the given height."""
return b'U' + pack('>I', height)
return UNDO_PREFIX + pack('>I', height)
def read_undo_info(self, height):
"""Read undo information from a file for the current height."""
return self.utxo_db.get(self.undo_key(height))
return self.db.get(self.undo_key(height))
def flush_undo_infos(self, batch_put, undo_infos):
"""undo_infos is a list of (undo_info, height) pairs."""
@@ -506,11 +787,7 @@ class LevelDB:
"""Returns a raw block read from disk. Raises FileNotFoundError
if the block isn't on-disk."""
def read():
with util.open_file(self.raw_block_path(height)) as f:
return f.read(-1)
return await asyncio.get_event_loop().run_in_executor(self.executor, read)
return await asyncio.get_event_loop().run_in_executor(self.executor, read_block_file, self.raw_block_path(height))
def write_raw_block(self, block, height):
"""Write a raw block to disk."""
@@ -525,17 +802,16 @@ class LevelDB:
def clear_excess_undo_info(self):
"""Clear excess undo info. Only most recent N are kept."""
prefix = b'U'
min_height = self.min_undo_height(self.db_height)
keys = []
for key, hist in self.utxo_db.iterator(prefix=prefix):
for key, hist in self.db.iterator(prefix=UNDO_PREFIX):
height, = unpack('>I', key[-4:])
if height >= min_height:
break
keys.append(key)
if keys:
with self.utxo_db.write_batch() as batch:
with self.db.write_batch() as batch:
for key in keys:
batch.delete(key)
self.logger.info(f'deleted {len(keys):,d} stale undo entries')
@@ -556,7 +832,7 @@ class LevelDB:
# -- UTXO database
def read_utxo_state(self):
state = self.utxo_db.get(b'state')
state = self.db.get(UTXO_STATE)
if not state:
self.db_height = -1
self.db_tx_count = 0
@@ -599,7 +875,7 @@ class LevelDB:
self.logger.info(f'height: {self.db_height:,d}')
self.logger.info(f'tip: {hash_to_hex_str(self.db_tip)}')
self.logger.info(f'tx count: {self.db_tx_count:,d}')
if self.utxo_db.for_sync:
if self.db.for_sync:
self.logger.info(f'flushing DB cache at {self.env.cache_MB:,d} MB')
if self.first_sync:
self.logger.info(f'sync time so far: {util.formatted_time(self.wall_time)}')
@@ -616,31 +892,18 @@ class LevelDB:
'first_sync': self.first_sync,
'db_version': self.db_version,
}
batch.put(b'state', repr(state).encode())
batch.put(UTXO_STATE, repr(state).encode())
def set_flush_count(self, count):
self.utxo_flush_count = count
with self.utxo_db.write_batch() as batch:
with self.db.write_batch() as batch:
self.write_utxo_state(batch)
async def all_utxos(self, hashX):
"""Return all UTXOs for an address sorted in no particular order."""
def read_utxos():
utxos = []
utxos_append = utxos.append
s_unpack = unpack
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
prefix = b'u' + hashX
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
value, = unpack('<Q', db_value)
tx_hash, height = self.fs_tx_hash(tx_num)
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
return utxos
while True:
utxos = await asyncio.get_event_loop().run_in_executor(self.executor, read_utxos)
utxos = [UTXO(tx_num, tx_pos, *self.fs_tx_hash(tx_num), value=value) for (tx_num, tx_pos, value) in utxos]
if all(utxo.tx_hash is not None for utxo in utxos):
return utxos
self.logger.warning(f'all_utxos: tx hash not '
@@ -653,45 +916,4 @@ class LevelDB:
Used by the mempool code.
"""
def lookup_hashXs():
"""Return (hashX, suffix) pairs, or None if not found,
for each prevout.
"""
def lookup_hashX(tx_hash, tx_idx):
idx_packed = pack('<H', tx_idx)
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:4] + idx_packed
# Find which entry, if any, the TX_HASH matches.
for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
tx_num_packed = db_key[-4:]
tx_num, = unpack('<I', tx_num_packed)
hash, height = self.fs_tx_hash(tx_num)
if hash == tx_hash:
return hashX, idx_packed + tx_num_packed
return None, None
return [lookup_hashX(*prevout) for prevout in prevouts]
def lookup_utxos(hashX_pairs):
def lookup_utxo(hashX, suffix):
if not hashX:
# This can happen when the daemon is a block ahead
# of us and has mempool txs spending outputs from
# that new block
return None
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
key = b'u' + hashX + suffix
db_value = self.utxo_db.get(key)
if not db_value:
# This can happen if the DB was updated between
# getting the hashXs and getting the UTXOs
return None
value, = unpack('<Q', db_value)
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
hashX_pairs = await asyncio.get_event_loop().run_in_executor(self.executor, lookup_hashXs)
return await asyncio.get_event_loop().run_in_executor(self.executor, lookup_utxos, hashX_pairs)
return await asyncio.get_event_loop().run_in_executor(self.executor, lookup_hashXs_utxos, prevouts)
+46 -41
View File
@@ -43,10 +43,12 @@ class Merkle:
def __init__(self, hash_func=double_sha256):
self.hash_func = hash_func
def tree_depth(self, hash_count):
return self.branch_length(hash_count) + 1
@staticmethod
def tree_depth(hash_count):
return Merkle.branch_length(hash_count) + 1
def branch_length(self, hash_count):
@staticmethod
def branch_length(hash_count):
"""Return the length of a merkle branch given the number of hashes."""
if not isinstance(hash_count, int):
raise TypeError('hash_count must be an integer')
@@ -54,7 +56,8 @@ class Merkle:
raise ValueError('hash_count must be at least 1')
return ceil(log(hash_count, 2))
def branch_and_root(self, hashes, index, length=None):
@staticmethod
def branch_and_root(hashes, index, length=None, hash_func=double_sha256):
"""Return a (merkle branch, merkle_root) pair given hashes, and the
index of one of those hashes.
"""
@@ -63,8 +66,8 @@ class Merkle:
raise TypeError('index must be an integer')
# This also asserts hashes is not empty
if not 0 <= index < len(hashes):
raise ValueError('index out of range')
natural_length = self.branch_length(len(hashes))
raise ValueError(f"index '{index}/{len(hashes)}' out of range")
natural_length = Merkle.branch_length(len(hashes))
if length is None:
length = natural_length
else:
@@ -73,7 +76,6 @@ class Merkle:
if length < natural_length:
raise ValueError('length out of range')
hash_func = self.hash_func
branch = []
for _ in range(length):
if len(hashes) & 1:
@@ -85,44 +87,47 @@ class Merkle:
return branch, hashes[0]
def root(self, hashes, length=None):
@staticmethod
def root(hashes, length=None):
"""Return the merkle root of a non-empty iterable of binary hashes."""
branch, root = self.branch_and_root(hashes, 0, length)
branch, root = Merkle.branch_and_root(hashes, 0, length)
return root
def root_from_proof(self, hash, branch, index):
"""Return the merkle root given a hash, a merkle branch to it, and
its index in the hashes array.
# @staticmethod
# def root_from_proof(hash, branch, index, hash_func=double_sha256):
# """Return the merkle root given a hash, a merkle branch to it, and
# its index in the hashes array.
#
# branch is an iterable sorted deepest to shallowest. If the
# returned root is the expected value then the merkle proof is
# verified.
#
# The caller should have confirmed the length of the branch with
# branch_length(). Unfortunately this is not easily done for
# bitcoin transactions as the number of transactions in a block
# is unknown to an SPV client.
# """
# for elt in branch:
# if index & 1:
# hash = hash_func(elt + hash)
# else:
# hash = hash_func(hash + elt)
# index >>= 1
# if index:
# raise ValueError('index out of range for branch')
# return hash
branch is an iterable sorted deepest to shallowest. If the
returned root is the expected value then the merkle proof is
verified.
The caller should have confirmed the length of the branch with
branch_length(). Unfortunately this is not easily done for
bitcoin transactions as the number of transactions in a block
is unknown to an SPV client.
"""
hash_func = self.hash_func
for elt in branch:
if index & 1:
hash = hash_func(elt + hash)
else:
hash = hash_func(hash + elt)
index >>= 1
if index:
raise ValueError('index out of range for branch')
return hash
def level(self, hashes, depth_higher):
@staticmethod
def level(hashes, depth_higher):
"""Return a level of the merkle tree of hashes the given depth
higher than the bottom row of the original tree."""
size = 1 << depth_higher
root = self.root
root = Merkle.root
return [root(hashes[n: n + size], depth_higher)
for n in range(0, len(hashes), size)]
def branch_and_root_from_level(self, level, leaf_hashes, index,
@staticmethod
def branch_and_root_from_level(level, leaf_hashes, index,
depth_higher):
"""Return a (merkle branch, merkle_root) pair when a merkle-tree has a
level cached.
@@ -146,10 +151,10 @@ class Merkle:
if not isinstance(leaf_hashes, list):
raise TypeError("leaf_hashes must be a list")
leaf_index = (index >> depth_higher) << depth_higher
leaf_branch, leaf_root = self.branch_and_root(
leaf_branch, leaf_root = Merkle.branch_and_root(
leaf_hashes, index - leaf_index, depth_higher)
index >>= depth_higher
level_branch, root = self.branch_and_root(level, index)
level_branch, root = Merkle.branch_and_root(level, index)
# Check last so that we know index is in-range
if leaf_root != level[index]:
raise ValueError('leaf hashes inconsistent with level')
@@ -191,7 +196,7 @@ class MerkleCache:
# Start from the beginning of any final partial segment.
# Retain the value of depth_higher; in practice this is fine
start = self._leaf_start(self.length)
hashes = await self.source_func(start, length - start)
hashes = self.source_func(start, length - start)
self.level[start >> self.depth_higher:] = self._level(hashes)
self.length = length
@@ -203,7 +208,7 @@ class MerkleCache:
level = self.level[:length >> self.depth_higher]
leaf_start = self._leaf_start(length)
count = min(self._segment_length(), length - leaf_start)
hashes = await self.source_func(leaf_start, count)
hashes = self.source_func(leaf_start, count)
level += self._level(hashes)
return level
@@ -211,7 +216,7 @@ class MerkleCache:
"""Call to initialize the cache to a source of given length."""
self.length = length
self.depth_higher = self.merkle.tree_depth(length) // 2
self.level = self._level(await self.source_func(0, length))
self.level = self._level(self.source_func(0, length))
self.initialized.set()
def truncate(self, length):
@@ -245,7 +250,7 @@ class MerkleCache:
await self._extend_to(length)
leaf_start = self._leaf_start(index)
count = min(self._segment_length(), length - leaf_start)
leaf_hashes = await self.source_func(leaf_start, count)
leaf_hashes = self.source_func(leaf_start, count)
if length < self._segment_length():
return self.merkle.branch_and_root(leaf_hashes, index)
level = await self._level_for(length)
+1 -1
View File
@@ -319,7 +319,7 @@ class PeerManager:
# Check prior header too in case of hard fork.
check_height = min(our_height, their_height)
raw_header = await self.db.raw_header(check_height)
raw_header = self.db.raw_header(check_height)
if ptuple >= (1, 4):
ours = raw_header.hex()
message = 'blockchain.block.header'
+51 -94
View File
@@ -17,7 +17,7 @@ from asyncio import Event, sleep
from collections import defaultdict
from functools import partial
from binascii import hexlify
from binascii import hexlify, unhexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from prometheus_client import Counter, Info, Histogram, Gauge
@@ -141,7 +141,11 @@ class SessionManager:
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version",))
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
labelnames=("method", "version"))
tx_request_count_metric = Counter("requested_transaction", "Number of transactions requested", namespace=NAMESPACE)
tx_replied_count_metric = Counter("replied_transaction", "Number of transactions responded", namespace=NAMESPACE)
urls_to_resolve_count_metric = Counter("urls_to_resolve", "Number of urls to resolve", namespace=NAMESPACE)
resolved_url_count_metric = Counter("resolved_url", "Number of resolved urls", namespace=NAMESPACE)
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
db_operational_error_metric = Counter(
@@ -603,7 +607,7 @@ class SessionManager:
async def raw_header(self, height):
"""Return the binary header at the given height."""
try:
return await self.db.raw_header(height)
return self.db.raw_header(height)
except IndexError:
raise RPCError(BAD_REQUEST, f'height {height:,d} '
'out of range') from None
@@ -1045,7 +1049,12 @@ class LBRYElectrumX(SessionBase):
async def claimtrie_resolve(self, *urls):
if urls:
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
count = len(urls)
try:
self.session_mgr.urls_to_resolve_count_metric.inc(count)
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
finally:
self.session_mgr.resolved_url_count_metric.inc(count)
async def get_server_height(self):
return self.bp.height
@@ -1218,10 +1227,7 @@ class LBRYElectrumX(SessionBase):
return await self.address_status(hashX)
async def hashX_unsubscribe(self, hashX, alias):
try:
del self.hashX_subs[hashX]
except ValueError:
pass
self.hashX_subs.pop(hashX, None)
def address_to_hashX(self, address):
try:
@@ -1268,11 +1274,11 @@ class LBRYElectrumX(SessionBase):
hashX = self.address_to_hashX(address)
return await self.hashX_unsubscribe(hashX, address)
async def get_balance(self, hashX):
utxos = await self.db.all_utxos(hashX)
confirmed = sum(utxo.value for utxo in utxos)
unconfirmed = await self.mempool.balance_delta(hashX)
return {'confirmed': confirmed, 'unconfirmed': unconfirmed}
# async def get_balance(self, hashX):
# utxos = await self.db.all_utxos(hashX)
# confirmed = sum(utxo.value for utxo in utxos)
# unconfirmed = await self.mempool.balance_delta(hashX)
# return {'confirmed': confirmed, 'unconfirmed': unconfirmed}
async def scripthash_get_balance(self, scripthash):
"""Return the confirmed and unconfirmed balance of a scripthash."""
@@ -1323,31 +1329,12 @@ class LBRYElectrumX(SessionBase):
f'require header height {height:,d} <= '
f'cp_height {cp_height:,d} <= '
f'chain height {max_height:,d}')
branch, root = await self.db.header_branch_and_root(cp_height + 1,
height)
branch, root = await self.db.header_branch_and_root(cp_height + 1, height)
return {
'branch': [hash_to_hex_str(elt) for elt in branch],
'root': hash_to_hex_str(root),
}
async def block_header(self, height, cp_height=0):
"""Return a raw block header as a hexadecimal string, or as a
dictionary with a merkle proof."""
height = non_negative_integer(height)
cp_height = non_negative_integer(cp_height)
raw_header_hex = (await self.session_mgr.raw_header(height)).hex()
if cp_height == 0:
return raw_header_hex
result = {'header': raw_header_hex}
result.update(await self._merkle_proof(cp_height, height))
return result
async def block_header_13(self, height):
"""Return a raw block header as a hexadecimal string.
height: the header's height"""
return await self.block_header(height)
async def block_headers(self, start_height, count, cp_height=0, b64=False):
"""Return count concatenated block headers as hex for the main chain;
starting at start_height.
@@ -1361,9 +1348,13 @@ class LBRYElectrumX(SessionBase):
max_size = self.MAX_CHUNK_SIZE
count = min(count, max_size)
headers, count = await self.db.read_headers(start_height, count)
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
headers, count = self.db.read_headers(start_height, count)
if b64:
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode()
else:
headers = headers.hex()
result = {
'base64' if b64 else 'hex': headers,
'count': count,
@@ -1381,7 +1372,7 @@ class LBRYElectrumX(SessionBase):
index = non_negative_integer(index)
size = self.coin.CHUNK_SIZE
start_height = index * size
headers, _ = await self.db.read_headers(start_height, size)
headers, _ = self.db.read_headers(start_height, size)
return headers.hex()
async def block_get_header(self, height):
@@ -1523,41 +1514,44 @@ class LBRYElectrumX(SessionBase):
f'network rules.\n\n{message}\n[{raw_tx}]')
async def transaction_info(self, tx_hash: str):
assert_tx_hash(tx_hash)
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
if not block_hash:
return raw_tx, {'block_height': -1}
merkle_height = (await self.daemon.deserialised_block(block_hash))['height']
merkle = await self.transaction_merkle(tx_hash, merkle_height)
return raw_tx, merkle
return (await self.transaction_get_batch(tx_hash))[tx_hash]
async def transaction_get_batch(self, *tx_hashes):
self.session_mgr.tx_request_count_metric.inc(len(tx_hashes))
if len(tx_hashes) > 100:
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
for tx_hash in tx_hashes:
assert_tx_hash(tx_hash)
batch_result = {}
batch_result = await self.db.fs_transactions(tx_hashes)
needed_merkles = {}
for tx_hash in tx_hashes:
if tx_hash in batch_result and batch_result[tx_hash][0]:
continue
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
merkle = {}
if block_hash:
block = await self.daemon.deserialised_block(block_hash)
height = block['height']
# print('lbrycrd txs', height, block['tx'])
try:
pos = block['tx'].index(tx_hash)
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
merkle["merkle"] = self._get_merkle_branch(block['tx'], pos)
merkle["pos"] = pos
needed_merkles[tx_hash] = raw_tx, block['tx'], pos, height
else:
height = -1
merkle['block_height'] = height
batch_result[tx_hash] = [raw_tx, merkle]
batch_result[tx_hash] = [raw_tx, {'block_height': -1}]
for tx_hash, (raw_tx, block_txs, pos, block_height) in needed_merkles.items():
batch_result[tx_hash] = raw_tx, {
'merkle': self._get_merkle_branch(block_txs, pos),
'pos': pos,
'block_height': block_height
}
self.session_mgr.tx_replied_count_metric.inc(len(tx_hashes))
return batch_result
async def transaction_get(self, tx_hash, verbose=False):
@@ -1572,19 +1566,6 @@ class LBRYElectrumX(SessionBase):
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
async def _block_hash_and_tx_hashes(self, height):
"""Returns a pair (block_hash, tx_hashes) for the main chain block at
the given height.
block_hash is a hexadecimal string, and tx_hashes is an
ordered list of hexadecimal strings.
"""
height = non_negative_integer(height)
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
block_hash = hex_hashes[0]
block = await self.daemon.deserialised_block(block_hash)
return block_hash, block['tx']
def _get_merkle_branch(self, tx_hashes, tx_pos):
"""Return a merkle branch to a transaction.
@@ -1604,35 +1585,11 @@ class LBRYElectrumX(SessionBase):
height: the height of the block it is in
"""
assert_tx_hash(tx_hash)
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
pos = tx_hashes.index(tx_hash)
except ValueError:
result = await self.transaction_get_batch(tx_hash)
if tx_hash not in result or result[tx_hash][1]['block_height'] <= 0:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
branch = self._get_merkle_branch(tx_hashes, pos)
return {"block_height": height, "merkle": branch, "pos": pos}
async def transaction_id_from_pos(self, height, tx_pos, merkle=False):
"""Return the txid and optionally a merkle proof, given
a block height and position in the block.
"""
tx_pos = non_negative_integer(tx_pos)
if merkle not in (True, False):
raise RPCError(BAD_REQUEST, f'"merkle" must be a boolean')
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
tx_hash = tx_hashes[tx_pos]
except IndexError:
raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in '
f'block {block_hash} at height {height:,d}')
if merkle:
branch = self._get_merkle_branch(tx_hashes, tx_pos)
return {"tx_hash": tx_hash, "merkle": branch}
else:
return tx_hash
f'block at height {height:,d}')
return result[tx_hash][1]
class LocalRPC(SessionBase):
+45 -31
View File
@@ -25,18 +25,18 @@ def db_class(db_dir, name):
class Storage:
"""Abstract base class of the DB backend abstraction."""
def __init__(self, db_dir, name, for_sync):
def __init__(self, db_dir, name, for_sync, read_only=False):
self.db_dir = db_dir
self.is_new = not os.path.exists(os.path.join(db_dir, name))
self.for_sync = for_sync or self.is_new
self.open(name, create=self.is_new)
self.open(name, create=self.is_new, read_only=read_only)
@classmethod
def import_module(cls):
"""Import the DB engine module."""
raise NotImplementedError
def open(self, name, create):
def open(self, name, create, read_only=False):
"""Open an existing database or create a new one."""
raise NotImplementedError
@@ -77,18 +77,16 @@ class LevelDB(Storage):
import plyvel
cls.module = plyvel
def open(self, name, create):
mof = 512 if self.for_sync else 128
def open(self, name, create, read_only=False):
mof = 10000
path = os.path.join(self.db_dir, name)
# Use snappy compression (the default)
self.db = self.module.DB(path, create_if_missing=create,
max_open_files=mof)
self.db = self.module.DB(path, create_if_missing=create, max_open_files=mof)
self.close = self.db.close
self.get = self.db.get
self.put = self.db.put
self.iterator = self.db.iterator
self.write_batch = partial(self.db.write_batch, transaction=True,
sync=True)
self.write_batch = partial(self.db.write_batch, transaction=True, sync=True)
class RocksDB(Storage):
@@ -99,17 +97,18 @@ class RocksDB(Storage):
import rocksdb
cls.module = rocksdb
def open(self, name, create):
mof = 512 if self.for_sync else 128
def open(self, name, create, read_only=False):
mof = 10000
path = os.path.join(self.db_dir, name)
# Use snappy compression (the default)
options = self.module.Options(create_if_missing=create,
use_fsync=True,
target_file_size_base=33554432,
max_open_files=mof)
self.db = self.module.DB(path, options)
self.db = self.module.DB(path, options, read_only=read_only)
self.get = self.db.get
self.put = self.db.put
self.multi_get = self.db.multi_get
def close(self):
# PyRocksDB doesn't provide a close method; hopefully this is enough
@@ -120,8 +119,8 @@ class RocksDB(Storage):
def write_batch(self):
return RocksDBWriteBatch(self.db)
def iterator(self, prefix=b'', reverse=False):
return RocksDBIterator(self.db, prefix, reverse)
def iterator(self, **kwargs):
return RocksDBIterator(self.db, **kwargs)
class RocksDBWriteBatch:
@@ -142,28 +141,43 @@ class RocksDBWriteBatch:
class RocksDBIterator:
"""An iterator for RocksDB."""
def __init__(self, db, prefix, reverse):
__slots__ = [
'start',
'prefix',
'stop',
'iterator',
'include_key',
'include_value',
'prev',
'reverse'
]
def __init__(self, db, prefix=None, start=None, stop=None, include_key=True, include_value=True, reverse=False):
self.start = start
self.prefix = prefix
if reverse:
self.iterator = reversed(db.iteritems())
nxt_prefix = util.increment_byte_string(prefix)
if nxt_prefix:
self.iterator.seek(nxt_prefix)
try:
next(self.iterator)
except StopIteration:
self.iterator.seek(nxt_prefix)
else:
self.iterator.seek_to_last()
else:
self.iterator = db.iteritems()
self.stop = stop
self.iterator = db.iteritems() if not reverse else reversed(db.iteritems())
if prefix is not None and start is None:
self.iterator.seek(prefix)
elif start is not None:
self.iterator.seek(start)
self.include_key = include_key
self.include_value = include_value
self.prev = None
self.reverse = reverse
def __iter__(self):
return self
def __next__(self):
k, v = next(self.iterator)
if not k.startswith(self.prefix):
if None not in (self.stop, self.prev) and self.prev.startswith(self.stop):
raise StopIteration
return k, v
self.prev, v = next(self.iterator)
prev = self.prev
if self.prefix is not None and not prev.startswith(self.prefix):
raise StopIteration
if self.include_key and self.include_value:
return prev, v
elif self.include_key:
return prev
return v
+8 -3
View File
@@ -145,14 +145,15 @@ class Wallet:
elif not self.is_locked:
log.warning(
"Disk encryption requested but no password available for encryption. "
"Saving wallet in an unencrypted state."
"Resetting encryption preferences and saving wallet in an unencrypted state."
)
self.preferences[ENCRYPT_ON_DISK] = False
return self.storage.write(self.to_dict())
@property
def hash(self) -> bytes:
h = sha256()
if self.preferences.get(ENCRYPT_ON_DISK, False):
if self.is_encrypted:
assert self.encryption_password is not None, \
"Encryption is enabled but no password is available, cannot generate hash."
h.update(self.encryption_password.encode())
@@ -219,7 +220,11 @@ class Wallet:
@property
def is_encrypted(self) -> bool:
return self.is_locked or self.preferences.get(ENCRYPT_ON_DISK, False)
# either its locked or it was unlocked using a password.
# if its set to encrypt on preferences but isnt encrypted and no password was given so far,
# then its not encrypted
return self.is_locked or (
self.preferences.get(ENCRYPT_ON_DISK, False) and self.encryption_password is not None)
def decrypt(self):
assert not self.is_locked, "Cannot decrypt a locked wallet, unlock first."
+1
View File
@@ -1,6 +1,7 @@
import curses
import time
import asyncio
import lbry.wallet
from lbry.conf import Config
from lbry.extras.daemon.client import daemon_rpc
+2 -1
View File
@@ -25,7 +25,8 @@ RETURN_DOCS = {
'File': encode_file_doc(),
'Transaction': encode_tx_doc(),
'Output': encode_txo_doc(),
'Address': 'an address in base58'
'Address': 'an address in base58',
'Dict': 'glorious data in dictionary',
}
+3 -1
View File
@@ -10,6 +10,8 @@ with open(os.path.join(BASE, 'README.md'), encoding='utf-8') as fh:
PLYVEL = []
if sys.platform.startswith('linux'):
PLYVEL.append('plyvel==1.0.5')
PLYVEL.append('python-rocksdb')
setup(
name=__name__,
@@ -34,7 +36,7 @@ setup(
},
install_requires=[
'aiohttp==3.5.4',
'aioupnp==0.0.17',
'aioupnp==0.0.18',
'appdirs==1.4.3',
'certifi>=2018.11.29',
'colorama==0.3.7',
@@ -2,6 +2,15 @@ import logging
import asyncio
from binascii import hexlify
from lbry.testcase import CommandTestCase
from lbry.wallet.server.leveldb import proc_ctx
def get_txids(height):
ctx = proc_ctx.get()
return [
ctx.ctx_tx_hash(tx_num)[0][::-1].hex()
for tx_num in range(ctx.tx_counts[height - 1], ctx.tx_counts[height])
]
class BlockchainReorganizationTests(CommandTestCase):
@@ -9,17 +18,26 @@ class BlockchainReorganizationTests(CommandTestCase):
VERBOSITY = logging.WARN
async def assertBlockHash(self, height):
self.assertEqual(
(await self.ledger.headers.hash(height)).decode(),
await self.blockchain.get_block_hash(height)
)
bp = self.conductor.spv_node.server.bp
block_hash = await self.blockchain.get_block_hash(height)
self.assertEqual(block_hash, (await self.ledger.headers.hash(height)).decode())
self.assertEqual(block_hash, (bp.db.fs_block_hashes(height, 1))[0][::-1].hex())
txids = await asyncio.get_event_loop().run_in_executor(bp.db.executor, get_txids, height)
txs = await bp.db.fs_transactions(txids)
block_txs = (await bp.daemon.deserialised_block(block_hash))['tx']
self.assertSetEqual(set(block_txs), set(txs.keys()), msg='leveldb/lbrycrd is missing transactions')
self.assertListEqual(block_txs, list(txs.keys()), msg='leveldb/lbrycrd transactions are of order')
async def test_reorg(self):
bp = self.conductor.spv_node.server.bp
bp.reorg_count_metric.set(0)
# invalidate current block, move forward 2
self.assertEqual(self.ledger.headers.height, 206)
await self.assertBlockHash(206)
height = 206
self.assertEqual(self.ledger.headers.height, height)
await self.assertBlockHash(height)
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
await self.blockchain.generate(2)
await self.ledger.on_header.where(lambda e: e.height == 207)
@@ -37,6 +55,11 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.assertBlockHash(207)
await self.assertBlockHash(208)
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
await self.blockchain.generate(3)
await self.ledger.on_header.where(lambda e: e.height == 211)
await self.assertBlockHash(209)
await self.assertBlockHash(210)
await self.assertBlockHash(211)
async def test_reorg_change_claim_height(self):
# sanity check
@@ -51,6 +74,8 @@ class BlockchainReorganizationTests(CommandTestCase):
# create a claim and verify it's returned by claim_search
self.assertEqual(self.ledger.headers.height, 207)
await self.assertBlockHash(207)
broadcast_tx = await self.daemon.jsonrpc_stream_create(
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
)
@@ -58,6 +83,8 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.generate(1)
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
self.assertEqual(self.ledger.headers.height, 208)
await self.assertBlockHash(208)
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertEqual(1, len(txos))
txo = txos[0]
@@ -75,17 +102,18 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.blockchain.clear_mempool()
await self.blockchain.generate(2)
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
# verify the claim was dropped from block 208 as returned by lbrycrdd
reorg_block_hash = await self.blockchain.get_block_hash(208)
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
block_207 = await self.blockchain.get_block(reorg_block_hash)
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
@@ -89,7 +89,7 @@ class ClaimSearchCommand(ClaimTestCase):
# 23829 claim ids makes the request just large enough
claim_ids = [
'0000000000000000000000000000000000000000',
] * 23829
] * 33829
with self.assertRaises(ConnectionResetError):
await self.claim_search(claim_ids=claim_ids)
@@ -262,6 +262,19 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
await self.assertFindsClaims([claim5], fee_currency='usd')
async def test_search_by_language(self):
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
claim2 = await self.stream_create('claim2', fee_amount='0.9', fee_currency='lbc')
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc', languages='en')
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc', languages='en')
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd', languages='es')
await self.assertFindsClaims([claim4, claim3], any_languages=['en'])
await self.assertFindsClaims([claim2, claim1], any_languages=['none'])
await self.assertFindsClaims([claim4, claim3, claim2, claim1], any_languages=['none', 'en'])
await self.assertFindsClaims([claim5], any_languages=['es'])
await self.assertFindsClaims([claim5, claim4, claim3], any_languages=['en', 'es'])
await self.assertFindsClaims([], fee_currency='foo')
async def test_search_by_channel(self):
@@ -319,6 +332,25 @@ class ClaimSearchCommand(ClaimTestCase):
not_channel_ids=[chan2_id], has_channel_signature=True, valid_channel_signature=True)
await match([], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True, valid_channel_signature=True)
async def test_limit_claims_per_channel(self):
match = self.assertFindsClaims
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
claim1 = await self.stream_create('claim1')
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
claim3 = await self.stream_create('claim3', channel_id=chan1_id)
claim4 = await self.stream_create('claim4', channel_id=chan1_id)
claim5 = await self.stream_create('claim5', channel_id=chan2_id)
claim6 = await self.stream_create('claim6', channel_id=chan2_id)
await match(
[claim6, claim5, claim4, claim3, claim1],
limit_claims_per_channel=2, claim_type='stream'
)
await match(
[claim6, claim5, claim4, claim3, claim2, claim1],
limit_claims_per_channel=3, claim_type='stream'
)
async def test_claim_type_and_media_type_search(self):
# create an invalid/unknown claim
address = await self.account.receiving.get_or_create_usable_address()
@@ -117,11 +117,6 @@ class ResolveCommand(BaseResolveTestCase):
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
# plain winning claim
await self.assertResolvesToClaimId('foo', claim_id3)
# sequence resolution
await self.assertResolvesToClaimId('foo:1', claim_id1)
await self.assertResolvesToClaimId('foo:2', claim_id2)
await self.assertResolvesToClaimId('foo:3', claim_id3)
await self.assertResolvesToClaimId('foo:4', None)
# amount order resolution
await self.assertResolvesToClaimId('foo$1', claim_id3)
await self.assertResolvesToClaimId('foo$2', claim_id2)
+300 -23
View File
@@ -1,5 +1,4 @@
import re
import time
import typing
from math import ceil
@@ -30,6 +29,17 @@ class MockedCommentServer:
'timestamp': None,
'channel_url': None,
'is_hidden': False,
'is_pinned': False,
}
REACT_SCHEMA = {
'react_id': None,
'comment_id': None,
'reaction_type': None,
'timestamp': None,
'signature': None,
'channel_id': None,
'channel_name': None
}
def __init__(self, port=2903):
@@ -39,7 +49,9 @@ class MockedCommentServer:
self.runner = None
self.server = None
self.comments = []
self.reacts = {}
self.comment_id = 0
self.react_id = 0
@classmethod
def _create_comment(cls, **kwargs):
@@ -47,6 +59,12 @@ class MockedCommentServer:
schema.update(**kwargs)
return schema
@classmethod
def _react(cls, **kwargs):
schema = cls.REACT_SCHEMA.copy()
schema.update(**kwargs)
return schema
def create_comment(self, claim_id=None, parent_id=None, channel_name=None, channel_id=None, **kwargs):
comment_id = self.comment_id
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
@@ -105,6 +123,21 @@ class MockedCommentServer:
return True
return False
def pin_comment(
self,
comment_id: typing.Union[int, str],
channel_name: str, channel_id: str, remove: bool,
signing_ts: str, signature: str
):
comment_id = self.get_comment_id(comment_id)
if self.is_signable(signature, signing_ts):
if remove:
self.comments[comment_id]['is_pinned'] = False
else:
self.comments[comment_id]['is_pinned'] = True
return self.comments[comment_id]
return False
def hide_comments(self, pieces: list):
hidden = []
for p in pieces:
@@ -117,27 +150,18 @@ class MockedCommentServer:
'visible': list(comment_ids - set(hidden))
}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
def get_claim_comments(self, claim_id, hidden=False, visible=False, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
if hidden ^ visible:
comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(comments)/page_size),
'total_items': len(comments),
'items': [self.clean(c) for c in (comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], comments)))
}
def get_claim_hidden_comments(self, claim_id, hidden=True, page=1, page_size=50):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
select_comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(select_comments) / page_size),
'total_items': len(select_comments),
'items': [self.clean(c) for c in (select_comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda c: c['is_hidden'], comments)))
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], self.comments)))
}
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
@@ -157,15 +181,99 @@ class MockedCommentServer:
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
def react(
self,
comment_ids=None,
channel_name=None,
channel_id=None,
remove=None,
clear_types=None,
type=None,
signing_ts=None,
**kwargs
):
comment_batch = comment_ids.split(',')
for item in comment_batch:
react_id = self.react_id
c_id = item.strip()
reacts_for_comment_id = [r for r in list(self.reacts.values()) if r['comment_id'] == c_id]
channels_reacts_for_comment_id = [r for r in reacts_for_comment_id if r['channel_id'] == channel_id]
if remove:
matching_react = None
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == type:
matching_react = reaction
break
if matching_react:
self.reacts.pop(int(matching_react['react_id']), None)
else:
if clear_types:
for r_type in clear_types.split(','):
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == r_type:
x = self.reacts.pop(int(reaction['react_id']), None)
react = self._react(
react_id=str(react_id),
comment_id=str(c_id),
channel_name=channel_name,
channel_id=channel_id,
reaction_type=type,
timestamp=str(int(time.time())),
**kwargs
)
self.reacts[self.react_id] = react
self.react_id += 1
self.clean(react)
return True
def list_reacts(self, comment_ids, channel_id, channel_name, types=None, **kwargs):
all_types = list(set([r['reaction_type'] for r in list(self.reacts.values())]))
comment_id_list = comment_ids.split(',')
# _reacts: {'a1': {'like': 0, 'dislike': 0}, 'a2': {'like': 0, 'dislike': 0}}
own_reacts = {}
other_reacts = {}
# for each comment_id
# add comment_id: {} to own_reacts and other_reacts
# for each react in own_reacts
# for each react in other_reacts
for cid in comment_id_list:
own_reacts[cid] = {}
other_reacts[cid] = {}
for r_type in all_types:
own_reacts[cid][r_type] = 0
other_reacts[cid][r_type] = 0
# process that comment ids reactions for own and other categories
reacts_for_comment = list(filter(lambda c: c['comment_id'] == cid, list(self.reacts.values())))
if types:
reacts_for_comment = list(filter(lambda c: c['reaction_type'] in types.split(','), reacts_for_comment))
own_reacts_for_comment = list(filter(lambda c: c['channel_id'] == channel_id, reacts_for_comment))
other_reacts_for_comment = list(filter(lambda c: c['channel_id'] != channel_id, reacts_for_comment))
if own_reacts_for_comment:
for react in own_reacts_for_comment:
own_reacts[cid][react['reaction_type']] += 1
if other_reacts_for_comment:
for react in other_reacts_for_comment:
other_reacts[cid][react['reaction_type']] += 1
return {
'my_reactions': own_reacts,
'others_reactions': other_reacts,
}
methods = {
'get_claim_comments': get_claim_comments,
'comment.List': get_claim_comments,
'get_comments_by_id': get_comments_by_id,
'create_comment': create_comment,
'abandon_comment': abandon_comment,
'get_channel_from_comment_id': get_comment_channel_by_id,
'get_claim_hidden_comments': get_claim_hidden_comments,
'hide_comments': hide_comments,
'edit_comment': edit_comment,
'comment.Create': create_comment,
'comment.Abandon': abandon_comment,
'comment.GetChannelFromCommentID': get_comment_channel_by_id,
'comment.Hide': hide_comments,
'comment.Pin': pin_comment,
'comment.Edit': edit_comment,
'reaction.React': react,
'reaction.List': list_reacts,
}
def process_json(self, body) -> dict:
@@ -427,6 +535,56 @@ class CommentCommands(CommandTestCase):
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test05_comment_pin(self):
# wherein a bee makes a sick burn on moth's channel and moth pins it
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
moth_stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
moth_claim_id = moth_stream['outputs'][0]['claim_id']
comment1 = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=moth_claim_id,
channel_id=bee['claim_id']
)
self.assertFalse(comment1['is_pinned'])
comment2 = await self.daemon.jsonrpc_comment_create(
comment='sick burn, brah',
claim_id=moth_claim_id,
channel_id=moth_id,
)
self.assertFalse(comment2['is_pinned'])
comments = await self.daemon.jsonrpc_comment_list(moth_claim_id)
comments_items = comments['items']
self.assertIn('is_pinned', comments_items[0])
self.assertFalse(comments_items[0]['is_pinned'])
pinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name']
)
self.assertTrue(pinned['is_pinned'])
comments_after_pinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_pin = comments_after_pinning['items']
self.assertTrue(items_after_pin[1]['is_pinned'])
unpinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
remove=True
)
self.assertFalse(unpinned['is_pinned'])
comments_after_unpinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_unpin = comments_after_unpinning['items']
self.assertFalse(items_after_unpin[1]['is_pinned'])
async def test06_comment_list(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
@@ -450,7 +608,6 @@ class CommentCommands(CommandTestCase):
channel_id=bee['claim_id']
)
all_comments = [other_comment, owner_comment, hidden_comment]
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
@@ -532,3 +689,123 @@ class CommentCommands(CommandTestCase):
comment='If you see it and you mean then you know you have to go',
comment_id=original_cid
)
async def test06_reactions(self):
# wherein a bee insulted by a rude moth accidentally
# 1) likes an insult
# 2) clicks dislike instead, testing "clear_types"
# 3) then clicks dislike again, testing "remove"
# calls from the point of view of the moth
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
claim_id = stream['outputs'][0]['claim_id']
first_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth['claim_id'],
)
second_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
all_comments = [second_comment, first_comment]
comment_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(comment_list)
)
self.assertEqual(comment_list['total_items'], 2)
bee_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='like',
)
moth_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_type='like',
)
reactions = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1}}, 'others_reactions': {'0': {'like': 1}}}
self.assertEqual(reactions['my_reactions']['0']['like'], 1)
self.assertEqual(reactions['others_reactions']['0']['like'], 1)
bee_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='dislike',
clear_types='like',
)
reactions_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1, 'dislike', 0}}, 'others_reactions': {'0': {'like': 0, 'dislike': 1}}}
self.assertEqual(reactions_after_bee_dislikes['my_reactions']['0']['like'], 1)
self.assertEqual(reactions_after_bee_dislikes['my_reactions']['0']['dislike'], 0)
self.assertEqual(reactions_after_bee_dislikes['others_reactions']['0']['dislike'], 1)
self.assertEqual(reactions_after_bee_dislikes['others_reactions']['0']['like'], 0)
only_likes_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_types='like',
)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions']['0']['like'], 1)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions']['0']['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions']['0']['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions']['0']['like'], 0)
bee_un_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
remove=True,
react_type='dislike',
)
reactions_after_bee_absconds = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_bee_absconds['my_reactions']['0']['like'], 1)
self.assertNotIn('dislike', reactions_after_bee_absconds['my_reactions']['0'])
self.assertEqual(reactions_after_bee_absconds['others_reactions']['0']['like'], 0)
self.assertNotIn('dislike', reactions_after_bee_absconds['others_reactions']['0'])
bee_reacts_to_both_comments = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'] + ',' + second_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='frozen_tom',
)
reactions_after_double_frozen_tom = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'] + ',' + second_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_double_frozen_tom['my_reactions']['0']['like'], 1)
self.assertNotIn('dislike', reactions_after_double_frozen_tom['my_reactions']['0'])
self.assertEqual(reactions_after_double_frozen_tom['others_reactions']['0']['frozen_tom'], 1)
self.assertEqual(reactions_after_double_frozen_tom['others_reactions']['1']['frozen_tom'], 1)
+5 -5
View File
@@ -37,8 +37,8 @@ class TestSigningComments(AsyncioTestCase):
rswanson = await get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
sign_comment(comment_body, rswanson, sign_comment_id=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, sign_comment_id=True))
async def test04_invalid_signature(self):
rswanson = await get_channel('@RonSwanson')
@@ -53,7 +53,7 @@ class TestSigningComments(AsyncioTestCase):
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
fake_abandon_signal = chair_comment.copy()
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
sign_comment(fake_abandon_signal, jeanralphio, sign_comment_id=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, sign_comment_id=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, sign_comment_id=True))
@@ -1,7 +1,9 @@
import binascii
import unittest
from lbry.dht.error import DecodeError
from lbry.dht.serialization.bencoding import _bencode
from lbry.dht.serialization.datagram import RequestDatagram, ResponseDatagram, decode_datagram, ErrorDatagram
from lbry.dht.serialization.datagram import _decode_datagram
from lbry.dht.serialization.datagram import REQUEST_TYPE, RESPONSE_TYPE, ERROR_TYPE
from lbry.dht.serialization.datagram import make_compact_address, decode_compact_address
@@ -139,6 +141,38 @@ class TestDatagram(unittest.TestCase):
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_str_or_int_keys(self):
datagram = decode_datagram(_bencode({
b'0': 0,
b'1': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
b'2': b'111111111111111111111111111111111111111111111111',
b'3': b'ping',
b'4': [{b'protocolVersion': 1}],
b'5': b'should not error'
}))
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_mixed_str_or_int_keys(self):
# datagram, _ = _bencode({
# b'0': 0,
# 1: b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
# b'2': b'111111111111111111111111111111111111111111111111',
# 3: b'ping',
# b'4': [{b'protocolVersion': 1}],
# b'5': b'should not error'
# }))
encoded = binascii.unhexlify(b"64313a3069306569316532303a0abcb5269d6cfc1e87a08e920bf39fe9df8e92fc313a3234383a313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131693365343a70696e67313a346c6431353a70726f746f636f6c56657273696f6e6931656565313a3531363a73686f756c64206e6f74206572726f7265")
self.assertDictEqual(
{
'packet_type': 0,
'rpc_id': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
'node_id': b'111111111111111111111111111111111111111111111111',
'method': b'ping',
'args': [{b'protocolVersion': 1}]
}, _decode_datagram(encoded)[0]
)
class TestCompactAddress(unittest.TestCase):
def test_encode_decode(self, address='1.2.3.4', port=4444, node_id=b'1' * 48):
+16 -15
View File
@@ -9,14 +9,15 @@ claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e"
class TestURLParsing(unittest.TestCase):
segments = 'stream', 'channel'
fields = 'name', 'claim_id', 'sequence', 'amount_order'
fields = 'name', 'claim_id', 'amount_order'
def _assert_url(self, url_string, **kwargs):
def _assert_url(self, url_string, strictly=True, **kwargs):
url = URL.parse(url_string)
if url_string.startswith('lbry://'):
self.assertEqual(url_string, str(url))
else:
self.assertEqual(f'lbry://{url_string}', str(url))
if strictly:
if url_string.startswith('lbry://'):
self.assertEqual(url_string, str(url))
else:
self.assertEqual(f'lbry://{url_string}', str(url))
present = {}
for key in kwargs:
for segment_name in self.segments:
@@ -42,19 +43,22 @@ class TestURLParsing(unittest.TestCase):
url = self._assert_url
# stream
url('test', stream_name='test')
url('test:1', stream_name='test', stream_sequence='1')
url('test*1', stream_name='test*1')
url('test$1', stream_name='test', stream_amount_order='1')
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id)
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id, strictly=False)
url(f'test:{claim_id}', stream_name='test', stream_claim_id=claim_id)
# channel
url('@test', channel_name='@test')
url('@test:1', channel_name='@test', channel_sequence='1')
url('@test$1', channel_name='@test', channel_amount_order='1')
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id)
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id, strictly=False)
url(f'@test:{claim_id}', channel_name='@test', channel_claim_id=claim_id)
# channel/stream
url('lbry://@test/stuff', channel_name='@test', stream_name='stuff')
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff', strictly=False)
url(f'lbry://@test:{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# combined legacy and new
url('@test:1/stuff#2', channel_claim_id='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False)
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\uD799')
@@ -104,10 +108,8 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://test@")
fail("lbry://tes@t")
fail(f"lbry://test:1#{claim_id}")
fail("lbry://test:0")
fail("lbry://test$0")
fail("lbry://test/path")
fail("lbry://@test1:1ab/fakepath")
fail("lbry://test:1:1:1")
fail("whatever/lbry://test")
fail("lbry://lbry://test")
@@ -115,5 +117,4 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://abc:0x123")
fail("lbry://abc:0x123/page")
fail("lbry://@test1#ABCDEF/fakepath")
fail("test:0001")
fail("lbry://@test1$1/fakepath?arg1&arg2&arg3")
+13 -6
View File
@@ -40,7 +40,7 @@ class TestReflector(AsyncioTestCase):
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream_manager.config.reflect_streams = False
self.stream = await self.stream_manager.create(file_path)
async def _test_reflect_stream(self, response_chunk_size):
@@ -93,8 +93,10 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
stop.set()
# this used to raise (and then propagate) a CancelledError
self.assertListEqual(await reflect_task, [])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertFalse(self.stream.is_fully_reflected)
self.assertFalse(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
async def test_result_from_disconnect_after_sd_transfer(self):
stop = asyncio.Event()
@@ -112,7 +114,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -134,7 +137,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -156,7 +160,9 @@ class TestReflector(AsyncioTestCase):
await not_incoming.wait()
await incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
# self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
@@ -181,7 +187,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await self.stream_manager.delete(self.stream, delete_file=True)
# this used to raise OSError when it can't read the deleted blob for the upload
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
+42
View File
@@ -74,6 +74,48 @@ class TestWalletCreation(AsyncioTestCase):
decrypted = Wallet.unpack('password', encrypted)
self.assertEqual(decrypted['accounts'][0]['name'], 'An Account')
def test_no_password_but_encryption_preferred(self):
wallet_dict = {
'version': 1,
'name': 'Main Wallet',
'preferences': {
"encrypt-on-disk": {
"ts": 1571762543.351794,
"value": True
},
},
'accounts': [
{
'certificates': {},
'name': 'An Account',
'ledger': 'lbc_mainnet',
'modified_on': 123,
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
"h absent",
'encrypted': False,
'private_key':
'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7'
'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe',
'public_key':
'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm'
'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9',
'address_generator': {
'name': 'deterministic-chain',
'receiving': {'gap': 17, 'maximum_uses_per_address': 3},
'change': {'gap': 10, 'maximum_uses_per_address': 3}
}
}
]
}
storage = WalletStorage(default=wallet_dict)
wallet = Wallet.from_storage(storage, self.manager)
self.assertEqual(
hexlify(wallet.hash), b'8cc6341885e6ad46f72a17364c65f8441f09e79996c55202196b399c75f8d751'
)
self.assertFalse(wallet.is_encrypted)
def test_read_write(self):
manager = WalletManager()
config = {'data_path': '/tmp/wallet'}