Compare commits

..
Author SHA1 Message Date
Lex Berezhny 24833ce9fb v0.83.0 2020-10-10 13:03:42 -04:00
Lex Berezhny ec2c18dc87 regenerate docs 2020-10-10 13:02:38 -04:00
jessop 7384609e74 support comment pinning 2020-10-10 13:02:38 -04:00
Lex Berezhny 3047649650 v0.82.0 2020-09-29 08:49:16 -04:00
kodxana d298dac3f3 Fixed docker-compose - no snapshot error
Curently new containers do not get provide snapshots from LBRYinc (Wallet and CRD) that causes containers to get into restart loop.
2020-09-29 08:43:49 -04:00
Lex Berezhny 1574bca8a8 Merge pull request #3054 from lbryio/ios-support
iOS support
2020-09-29 08:42:17 -04:00
Lex Berezhny ec2f6c6b80 lint 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola 838cc60161 conditional import 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola 310c61a5cc check KIVY_BUILD env variable 2020-09-29 07:52:33 -04:00
Lex Berezhny 318cc15323 Merge pull request #3045 from lbryio/bencode-byte-keys
add forward compatibility for byte datagram keys
2020-09-29 07:23:49 -04:00
Jack Robison 3a64ceb4d6 add forward compatibility for byte datagram keys 2020-09-28 15:56:13 -04:00
Lex Berezhny d0f21c0095 Merge pull request #3050 from lbryio/language-indexes
add indexes for `any_languages` argument to `claim_search`
2020-09-28 15:52:41 -04:00
Jack Robison 46dc15dd29 use language indexes 2020-09-28 14:45:51 -04:00
Jack Robison 8dc654b513 add language table and indexes 2020-09-28 14:45:51 -04:00
Jack Robison 7000ac3f3f update indexed tags 2020-09-28 14:45:51 -04:00
Lex Berezhny 43c2e8d8e9 Merge pull request #3051 from lbryio/comments-v2
Comments v2 with Reactions
2020-09-28 14:44:54 -04:00
jessop 0231139b01 support anonymous react_list 2020-09-27 16:36:51 -04:00
jessop d6ee6446dd react list takes comment_ids 2020-09-26 15:21:26 -04:00
Lex Berezhny 7b666efcf8 lbrycrd download path 2020-09-24 20:11:40 -04:00
Lex Berezhny eb5d2198fc lint 2020-09-24 20:09:46 -04:00
Lex Berezhny 34e44ebd1c minor cleanup and lint fixes 2020-09-24 20:03:22 -04:00
jessop bf2f4bc040 wip 2020-09-24 17:47:59 -04:00
jessop 9dc4559aba new comment and reactions api 2020-09-23 16:43:28 -04:00
Lex Berezhny eba8856261 v0.81.0 2020-08-19 15:47:53 -04:00
Lex Berezhny a8c5aa471a update docs 2020-08-19 15:46:29 -04:00
Lex Berezhny 52f6dcf092 Merge pull request #3021 from lbryio/debug-reflector
Improve reflector upload cancellation handling
2020-08-19 15:44:27 -04:00
Jack Robison dec79f3742 pylint 2020-08-19 15:24:17 -04:00
Jack Robison 8bdcac0f3e close reader before deleting blob 2020-08-19 15:24:17 -04:00
Jack Robison 8426b674a3 improve reflector upload cancellation handling 2020-08-19 15:24:17 -04:00
Lex Berezhny da391bcc8d Merge pull request #3028 from lbryio/limit_claims_per_channel
added `--limit_claims_per_channel` argument to `claim_search` to only return up to the specified number of claims per channel
2020-08-19 15:22:53 -04:00
Lex Berezhny 2d7443d454 slight cleanup 2020-08-19 13:59:26 -04:00
Lex Berezhny 991987ed76 docopt fix 2020-08-19 12:08:41 -04:00
Lex Berezhny ec24ebf2cf added --limit_claims_per_channel argument to claim_search to only return up to the specified number of claims per channel 2020-08-19 10:51:31 -04:00
Lex Berezhny 6ba0976085 Merge pull request #3022 from btzr-io/patch-2
Add webvtt mime type ( subtitles, captions )
2020-08-12 18:58:07 -04:00
Baltazar Gomez 2b88d01a01 Add webvtt mime type ( subtitles, captions ) 2020-08-12 02:25:01 -05:00
Lex Berezhny 0c09f24cbf v0.80.0 2020-08-04 16:53:52 -04:00
Lex Berezhny 61d22afeba Merge pull request #3016 from lbryio/release_all_outputs_on_start
release reserved outputs on startup
2020-08-04 16:53:15 -04:00
Lex Berezhny 9f1ed6e8c3 release_all_outputs on startup 2020-08-04 15:03:58 -04:00
Lex Berezhny bbc4113cac Merge pull request #3015 from lbryio/new_sdk_forwarding
added support for using the new SDK for `resolve` and `claim_search` by passing `--new_sdk_server`
2020-08-04 15:03:39 -04:00
Lex Berezhny 91194bf422 doc fix 2020-08-04 14:12:40 -04:00
Lex Berezhny 9c5f940b00 claim search forwarding to new sdk 2020-08-04 12:11:02 -04:00
Lex Berezhny 455b4043b8 new resolve 2020-08-04 11:33:39 -04:00
25 changed files with 1474 additions and 475 deletions

No files matched your search

+4 -2
View File
@@ -16,7 +16,8 @@ services:
- "lbrycrd:/data/.lbrycrd"
environment:
- RUN_MODE=default
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
# Curently not snapshot provided
#- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
- RPC_ALLOW_IP=0.0.0.0/0
wallet_server:
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
@@ -30,5 +31,6 @@ services:
volumes:
- "wallet_server:/database"
environment:
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
# Curently not snapshot provided
# - SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
+437 -167
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.79.1"
__version__ = "0.83.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+1 -1
View File
@@ -337,9 +337,9 @@ class BlobFile(AbstractBlob):
return self.loop.create_task(write_blob())
def delete(self):
super().delete()
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
@classmethod
async def create_from_unencrypted(
+1 -1
View File
@@ -611,7 +611,7 @@ class Config(CLIConfig):
('lbrynet4.lbry.com', 4444) # ASIA
])
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
+17 -7
View File
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
self.response = response.decode()
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
def _decode_datagram(datagram: bytes):
msg_types = {
REQUEST_TYPE: RequestDatagram,
RESPONSE_TYPE: ResponseDatagram,
@@ -152,19 +152,29 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
}
primitive: typing.Dict = bdecode(datagram)
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
converted = {
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
}
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
else:
raise ValueError("invalid datagram type")
datagram_class = msg_types[datagram_type]
decoded = {
k: primitive[i] # pylint: disable=unsubscriptable-object
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
for i, k in enumerate(datagram_class.required_fields)
if i in primitive # pylint: disable=unsupported-membership-test
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
}
for i, _ in enumerate(OPTIONAL_FIELDS):
if i + OPTIONAL_ARG_OFFSET in primitive:
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
return decoded, datagram_class
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
decoded, datagram_class = _decode_datagram(datagram)
return datagram_class(**decoded)
+15 -5
View File
@@ -22,10 +22,10 @@ def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
if isinstance(channel, Output):
try:
signing_field = comment['comment_id'] if abandon else comment['comment']
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
pieces = [
comment['signing_ts'].encode(),
cid2hash(comment['channel_id']),
@@ -41,9 +41,9 @@ def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
return False
def sign_comment(comment: dict, channel: Output, abandon=False):
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if abandon else comment['comment']
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
@@ -52,11 +52,21 @@ def sign_comment(comment: dict, channel: Output, abandon=False):
'signing_ts': timestamp
})
def sign_reaction(reaction: dict, channel: Output):
timestamp = str(int(time.time()))
signing_field = reaction['channel_name']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
reaction.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
async with utils.aiohttp_request('POST', url, json=json_body) as response:
try:
result = await response.json()
+175 -17
View File
@@ -986,10 +986,12 @@ class Daemon(metaclass=JSONRPCServerType):
[--include_sent_supports]
[--include_sent_tips]
[--include_received_tips]
[--new_sdk_server=<new_sdk_server>]
Options:
--urls=<urls> : (str, list) one or more urls to resolve
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase reciepts
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
--include_purchase_receipt : (bool) lookup and include a receipt if this wallet
has purchased the claim being resolved
--include_is_my_output : (bool) lookup and include a boolean indicating
@@ -2308,6 +2310,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--channel=<channel> |
[[--channel_ids=<channel_ids>...] [--not_channel_ids=<not_channel_ids>...]]]
[--has_channel_signature] [--valid_channel_signature | --invalid_channel_signature]
[--limit_claims_per_channel=<limit_claims_per_channel>]
[--is_controlling] [--release_time=<release_time>] [--public_key_id=<public_key_id>]
[--timestamp=<timestamp>] [--creation_timestamp=<creation_timestamp>]
[--height=<height>] [--creation_height=<creation_height>]
@@ -2327,6 +2330,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--not_locations=<not_locations>...]
[--order_by=<order_by>...] [--page=<page>] [--page_size=<page_size>]
[--wallet_id=<wallet_id>] [--include_purchase_receipt] [--include_is_my_output]
[--new_sdk_server=<new_sdk_server>]
Options:
--name=<name> : (str) claim name (normalized)
@@ -2355,6 +2359,8 @@ class Daemon(metaclass=JSONRPCServerType):
--invalid_channel_signature : (bool) claims with invalid channel signature or no signature,
use in conjunction with --has_channel_signature to
only get claims with invalid signatures
--limit_claims_per_channel=<limit_claims_per_channel>: (int) only return up to the specified
number of claims per channel
--is_controlling : (bool) winning claims of their respective name
--public_key_id=<public_key_id> : (str) only return channels having this public key id, this is
the same key as used in the wallet file to map
@@ -2432,6 +2438,7 @@ class Daemon(metaclass=JSONRPCServerType):
has purchased the claim
--include_is_my_output : (bool) lookup and include a boolean indicating
if claim being resolved is yours
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
Returns: {Paginated[Output]}
"""
@@ -5041,8 +5048,9 @@ class Daemon(metaclass=JSONRPCServerType):
if hidden ^ visible:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_claim_hidden_comments',
'comment.List',
claim_id=claim_id,
visible=visible,
hidden=hidden,
page=page,
page_size=page_size
@@ -5050,7 +5058,7 @@ class Daemon(metaclass=JSONRPCServerType):
else:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_claim_comments',
'comment.List',
claim_id=claim_id,
parent_id=parent_id,
page=page,
@@ -5058,15 +5066,11 @@ class Daemon(metaclass=JSONRPCServerType):
top_level=not include_replies
)
if not skip_validation:
channel_urls = []
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if channel_url:
channel_urls.append(channel_url)
resolve_response = await self.resolve([], channel_urls)
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if not channel_url:
continue
resolve_response = await self.resolve([], [channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
comment, resolve_response[channel_url]
@@ -5106,8 +5110,10 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"is_pinned": (boolean) Channel owner has pinned this comment,
"signature": (str) The signature of the comment,
"signing_ts": (str) The timestamp used to sign the comment,
"channel_url": (str) Channel's URI in the ClaimTrie,
@@ -5129,7 +5135,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Create', comment_body)
response.update({
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
})
@@ -5155,18 +5161,20 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"signature": (str) The signature of the comment,
"signing_ts": (str) Timestamp used to sign the most recent signature,
"channel_url": (str) Channel's URI in the ClaimTrie,
"is_pinned": (boolean) Channel owner has pinned this comment,
"parent_id": (str) Comment this is replying to, (None) if this is the root,
"timestamp": (int) The time at which comment was entered into the server at, in nanoseconds.
}
"""
channel = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_channel_from_comment_id',
'comment.GetChannelFromCommentID',
comment_id=comment_id
)
if 'error' in channel:
@@ -5183,7 +5191,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(edited_comment, channel_claim)
return await comment_client.jsonrpc_post(
self.conf.comment_server, 'edit_comment', edited_comment
self.conf.comment_server, 'comment.Edit', edited_comment
)
@requires(WALLET_COMPONENT)
@@ -5209,7 +5217,7 @@ class Daemon(metaclass=JSONRPCServerType):
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
abandon_comment_body = {'comment_id': comment_id}
channel = await comment_client.jsonrpc_post(
self.conf.comment_server, 'get_channel_from_comment_id', comment_id=comment_id
self.conf.comment_server, 'comment.GetChannelFromCommentID', comment_id=comment_id
)
if 'error' in channel:
return {comment_id: {'abandoned': False}}
@@ -5218,8 +5226,8 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(abandon_comment_body, channel, abandon=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'abandon_comment', abandon_comment_body)
comment_client.sign_comment(abandon_comment_body, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Abandon', abandon_comment_body)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_hide(self, comment_ids: typing.Union[str, list], wallet_id=None):
@@ -5263,9 +5271,159 @@ class Daemon(metaclass=JSONRPCServerType):
for_signing=True
)
piece = {'comment_id': comment['comment_id']}
comment_client.sign_comment(piece, channel, abandon=True)
comment_client.sign_comment(piece, channel, sign_comment_id=True)
pieces.append(piece)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'hide_comments', pieces=pieces)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Hide', pieces=pieces)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_pin(self, comment_id=None, channel_id=None, channel_name=None,
channel_account_id=None, remove=False, wallet_id=None):
"""
Pin a comment published to a claim you control.
Usage:
comment_pin (<comment_id> | --comment_id=<comment_id>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
[--remove]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_id=<comment_id> : (str) Hash identifying the comment to pin
--channel_id=<claim_id> : (str) The ID of channel owning the commented claim
--channel_name=<claim_name> : (str) The name of channel owning the commented claim
--remove : (bool) remove the pin
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--wallet_id=<wallet_id : (str) restrict operation to specific wallet
Returns: lists containing the ids comments that are hidden and visible.
{
"hidden": (list) IDs of hidden comments.
"visible": (list) IDs of visible comments.
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
comment_pin_args = {
'comment_id': comment_id,
'channel_name': channel_name,
'channel_id': channel_id,
'remove': remove,
}
comment_client.sign_comment(comment_pin_args, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Pin', comment_pin_args)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, remove=False, clear_types=None, react_type=None, wallet_id=None):
"""
Create and associate a reaction emoji with a comment using your channel identity.
Usage:
comment_react (--comment_ids=<comment_ids>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
(--react_type=<react_type>)
[(--remove) | (--clear_types=<clear_types>)]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_ids=<comment_ids> : (str) one or more comment id reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--react_type=<react_type> : (str) name of reaction type
--remove : (bool) remove specified react_type
--clear_types=<clear_types> : (str) types to clear when adding another type
Returns:
(dict) Reaction object if successfully made, (None) otherwise
{
"Reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction
...
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_body = {
'comment_ids': comment_ids,
'channel_id': channel_id,
'channel_name': channel.claim_name,
'type': react_type,
'remove': remove,
'clear_types': clear_types,
}
comment_client.sign_reaction(react_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.React', react_body)
return response
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react_list(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, react_types=None, wallet_id=None):
"""
List reactions emoji with a claim using your channel identity.
Usage:
comment_react_list (--comment_ids=<comment_ids>)
[(--channel_id=<channel_id>)(--channel_name=<channel_name>)]
[--react_types=<react_types>]
Options:
--comment_ids=<comment_ids> : (str) The comment ids reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--react_types=<react_type> : (str) comma delimited reaction types
Returns:
(dict) Comment object if successfully made, (None) otherwise
{
"my_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
"other_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
react_list_body = {
'comment_ids': comment_ids,
}
if channel_id:
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_list_body['channel_id'] = channel_id
react_list_body['channel_name'] = channel.claim_name
if react_types:
react_list_body['types'] = react_types
if channel_id:
comment_client.sign_reaction(react_list_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.List', react_list_body)
return response
async def broadcast_or_release(self, tx, blocking=False):
await self.wallet_manager.broadcast_or_release(tx, blocking)
+1
View File
@@ -123,6 +123,7 @@ types_map = {
'.txt': ('text/plain', 'document'),
'.ustar': ('application/x-ustar', 'binary'),
'.vcf': ('text/x-vcard', 'document'),
'.vtt': ('text/vtt', 'document'),
'.wav': ('audio/x-wav', 'audio'),
'.webm': ('video/webm', 'video'),
'.wiz': ('application/msword', 'document'),
+9 -2
View File
@@ -25,12 +25,14 @@ def set_reference(reference, claim_hash, rows):
class Censor:
__slots__ = 'streams', 'channels', 'censored', 'total'
__slots__ = 'streams', 'channels', 'limit_claims_per_channel', 'censored', 'claims_in_channel', 'total'
def __init__(self, streams: dict = None, channels: dict = None):
def __init__(self, streams: dict = None, channels: dict = None, limit_claims_per_channel: int = None):
self.streams = streams or {}
self.channels = channels or {}
self.limit_claims_per_channel = limit_claims_per_channel # doesn't count as censored
self.censored = {}
self.claims_in_channel = {}
self.total = 0
def censor(self, row) -> bool:
@@ -49,6 +51,11 @@ class Censor:
break
if was_censored:
self.total += 1
if not was_censored and self.limit_claims_per_channel is not None and row['channel_hash']:
self.claims_in_channel.setdefault(row['channel_hash'], 0)
self.claims_in_channel[row['channel_hash']] += 1
if self.claims_in_channel[row['channel_hash']] > self.limit_claims_per_channel:
return True
return was_censored
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
+13 -3
View File
@@ -356,13 +356,23 @@ class ManagedStream(ManagedDownloadSource):
return sent
except ConnectionRefusedError:
return sent
except OSError:
# raised if a blob is deleted while it's being sent
return sent
except (OSError, Exception) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
raise err
if isinstance(err, OSError):
log.warning(
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
self.claim_id
)
else:
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
raise asyncio.CancelledError()
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+22 -6
View File
@@ -64,12 +64,12 @@ class StreamReflectorClient(asyncio.Protocol):
self.transport.write(msg.encode())
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
return await self.pending_request
except (AttributeError, asyncio.CancelledError):
except (AttributeError, asyncio.CancelledError) as err:
# attribute error happens when we transport.write after disconnect
# cancelled error happens when the pending_request task is cancelled by a disconnect
if self.transport:
self.transport.close()
raise asyncio.TimeoutError()
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
finally:
self.pending_request = None
@@ -94,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
needed = response.get('needed_blobs', [])
sent_sd = False
if response['send_sd_blob']:
await sd_blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
try:
sent = await sd_blob.sendfile(self)
if sent == -1:
log.warning("failed to send sd blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_sd_blob'):
sent_sd = True
if not needed:
@@ -118,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
if 'send_blob' not in response:
raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']:
await blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
try:
sent = await blob.sendfile(self)
if sent == -1:
log.warning("failed to send blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8])
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+16 -15
View File
@@ -7,8 +7,6 @@ from binascii import hexlify
from collections import defaultdict
from dataclasses import dataclass
from contextvars import ContextVar
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
from datetime import date
from prometheus_client import Gauge, Counter, Histogram
@@ -19,6 +17,12 @@ from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Inpu
from .constants import TXO_TYPES, CLAIM_TYPES
from .util import date_to_julian_day
from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' or 'KIVY_BUILD' in os.environ:
from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported
else:
from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass
log = logging.getLogger(__name__)
sqlite3.enable_callback_tracebacks(True)
@@ -62,12 +66,6 @@ def run_read_only_fetchone(sql, params):
raise
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' in os.environ:
ReaderExecutorClass = ThreadPoolExecutor
else:
ReaderExecutorClass = ProcessPoolExecutor
class AIOSQLite:
reader_executor: ReaderExecutorClass
@@ -1254,13 +1252,16 @@ class Database(SQLiteMixin):
self.constrain_collections(constraints)
return self.get_utxo_count(**constraints)
async def release_all_outputs(self, account):
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
async def release_all_outputs(self, account=None):
if account is None:
await self.db.execute_fetchall("UPDATE txo SET is_reserved = 0 WHERE is_reserved = 1")
else:
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
def get_supports_summary(self, read_only=False, **constraints):
return self.get_txos(
+12 -4
View File
@@ -331,6 +331,7 @@ class Ledger(metaclass=LedgerRegistry):
async with self._header_processing_lock:
await self._update_tasks.add(self.initial_headers_sync())
await fully_synced
await self.db.release_all_outputs()
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
if len(self.accounts) > 10:
@@ -922,8 +923,11 @@ class Ledger(metaclass=LedgerRegistry):
txo.received_tips = tips
return txos, blocked, outputs.offset, outputs.total
async def resolve(self, accounts, urls, **kwargs):
resolve = partial(self.network.retriable_call, self.network.resolve)
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
if new_sdk_server:
resolve = partial(self.network.new_resolve, new_sdk_server)
else:
resolve = partial(self.network.retriable_call, self.network.resolve)
urls_copy = list(urls)
txos = []
while urls_copy:
@@ -943,9 +947,13 @@ class Ledger(metaclass=LedgerRegistry):
async def claim_search(
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
**kwargs) -> Tuple[List[Output], dict, int, int]:
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
if new_sdk_server:
claim_search = partial(self.network.new_claim_search, new_sdk_server)
else:
claim_search = self.network.claim_search
return await self._inflate_outputs(
self.network.claim_search(**kwargs), accounts,
claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output
)
+19
View File
@@ -5,6 +5,8 @@ from time import perf_counter
from operator import itemgetter
from typing import Dict, Optional, Tuple
import aiohttp
from lbry import __version__
from lbry.error import IncompatibleWalletServerError
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
@@ -181,6 +183,8 @@ class Network:
'blockchain.address.subscribe': self._on_status_controller,
}
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
@property
def config(self):
return self.ledger.config
@@ -207,6 +211,7 @@ class Network:
async def start(self):
self.running = True
self.aiohttp_session = aiohttp.ClientSession()
self._switch_task = asyncio.ensure_future(self.switch_forever())
# this may become unnecessary when there are no more bugs found,
# but for now it helps understanding log reports
@@ -217,6 +222,7 @@ class Network:
async def stop(self):
if self.running:
self.running = False
await self.aiohttp_session.close()
self._switch_task.cancel()
self.session_pool.stop()
@@ -316,6 +322,19 @@ class Network:
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
async def new_resolve(self, server, urls):
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
async def new_claim_search(self, server, **kwargs):
kwargs['protobuf'] = True
message = {"method": "claim_search", "params": kwargs}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
class SessionPool:
+1
View File
@@ -768,6 +768,7 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
+297 -200
View File
@@ -13,209 +13,306 @@ STREAM_TYPES = {
'model': 6
}
# 9/21/2020
MOST_USED_TAGS = {
"gaming",
"people & blogs",
"entertainment",
"music",
"pop culture",
"education",
"technology",
"blockchain",
"news",
"funny",
"science & technology",
"learning",
"gameplay",
"news & politics",
"comedy",
"bitcoin",
"beliefs",
"nature",
"art",
"economics",
"film & animation",
"lets play",
"games",
"sports",
"howto & style",
"game",
"cryptocurrency",
"playstation 4",
"automotive",
"crypto",
"mature",
"sony interactive entertainment",
"walkthrough",
"tutorial",
"video game",
"weapons",
"pc",
"playthrough",
"anime",
"how to",
"btc",
"fun",
"ethereum",
"food",
"travel & events",
"minecraft",
"science",
"autos & vehicles",
"play",
"politics",
"commentary",
"twitch",
"ps4live",
"love",
"ps4",
"nonprofits & activism",
"ps4share",
"fortnite",
"xbox",
"porn",
"video games",
"trump",
"español",
"money",
"music video",
"movie",
"coronavirus",
"nintendo",
"donald trump",
"steam",
"trailer",
"android",
"podcast",
"xbox one",
"survival",
"linux",
"travel",
"funny moments",
"audio",
"litecoin",
"animation",
"gamer",
"lets",
"playstation",
"bitcoin news",
"history",
"fox news",
"xxx",
"god",
"dance",
"adventure",
"liberal",
"horror",
"government",
"freedom",
"2020",
"reaction",
"meme",
"photography",
"truth"
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
def normalize_tag(tag):
return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
}
INDEXED_LANGUAGES = [
'en',
'aa',
'ab',
'ae',
'af',
'ak',
'am',
'an',
'ar',
'as',
'av',
'ay',
'az',
'ba',
'be',
'bg',
'bh',
'bi',
'bm',
'bn',
'bo',
'br',
'bs',
'ca',
'ce',
'ch',
'co',
'cr',
'cs',
'cu',
'cv',
'cy',
'da',
'de',
'dv',
'dz',
'ee',
'el',
'eo',
'es',
'et',
'eu',
'fa',
'ff',
'fi',
'fj',
'fo',
'fr',
'fy',
'ga',
'gd',
'gl',
'gn',
'gu',
'gv',
'ha',
'he',
'hi',
'ho',
'hr',
'ht',
'hu',
'hy',
'hz',
'ia',
'id',
'ie',
'ig',
'ii',
'ik',
'io',
'is',
'it',
'iu',
'ja',
'jv',
'ka',
'kg',
'ki',
'kj',
'kk',
'kl',
'km',
'kn',
'ko',
'kr',
'ks',
'ku',
'kv',
'kw',
'ky',
'la',
'lb',
'lg',
'li',
'ln',
'lo',
'lt',
'lu',
'lv',
'mg',
'mh',
'mi',
'mk',
'ml',
'mn',
'mr',
'ms',
'mt',
'my',
'na',
'nb',
'nd',
'ne',
'ng',
'nl',
'nn',
'no',
'nr',
'nv',
'ny',
'oc',
'oj',
'om',
'or',
'os',
'pa',
'pi',
'pl',
'ps',
'pt',
'qu',
'rm',
'rn',
'ro',
'ru',
'rw',
'sa',
'sc',
'sd',
'se',
'sg',
'si',
'sk',
'sl',
'sm',
'sn',
'so',
'sq',
'sr',
'ss',
'st',
'su',
'sv',
'sw',
'ta',
'te',
'tg',
'th',
'ti',
'tk',
'tl',
'tn',
'to',
'tr',
'ts',
'tt',
'tw',
'ty',
'ug',
'uk',
'ur',
'uz',
've',
'vi',
'vo',
'wa',
'wo',
'xh',
'yi',
'yo',
'za',
'zh',
'zu'
]
+18 -5
View File
@@ -18,7 +18,7 @@ from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs, Censor
from lbry.wallet import Ledger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
from .full_text_search import FTS_ORDER_BY
@@ -39,7 +39,7 @@ ATTRIBUTE_ARRAY_MAX_LENGTH = 100
INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
'tx_position', 'channel_join', 'reposted',
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed',
'trending_local', 'trending_global',
@@ -96,8 +96,8 @@ class ReaderState:
def get_resolve_censor(self) -> Censor:
return Censor(self.blocked_streams, self.blocked_channels)
def get_search_censor(self) -> Censor:
return Censor(self.filtered_streams, self.filtered_channels)
def get_search_censor(self, limit_claims_per_channel: int) -> Censor:
return Censor(self.filtered_streams, self.filtered_channels, limit_claims_per_channel)
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
@@ -421,12 +421,13 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
assert set(constraints).issubset(SEARCH_PARAMS), \
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
total = None
limit_claims_per_channel = constraints.pop('limit_claims_per_channel', None)
if not constraints.pop('no_totals', False):
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
context = ctx.get()
search_censor = context.get_search_censor()
search_censor = context.get_search_censor(limit_claims_per_channel)
txo_rows = search_claims(search_censor, **constraints)
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
@@ -535,6 +536,18 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
AND tag IN ({values})
)
"""
elif attr == 'language':
indexed_languages = any_items & set(INDEXED_LANGUAGES)
if indexed_languages:
any_items -= indexed_languages
for language in indexed_languages:
any_queries[f'#_any_common_languages_{language}'] = f"""
EXISTS(
SELECT 1 FROM language INDEXED BY language_{language}_idx
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=language.claim_hash
AND language = '{language}'
)
"""
if any_items:
+32 -4
View File
@@ -6,7 +6,6 @@ from decimal import Decimal
from collections import namedtuple
from multiprocessing import Manager
from binascii import unhexlify
from lbry.wallet.server.leveldb import LevelDB
from lbry.wallet.server.util import class_logger
from lbry.wallet.database import query, constraints_to_sql
@@ -19,7 +18,7 @@ from lbry.wallet.server.db.canonical import register_canonical_functions
from lbry.wallet.server.db.full_text_search import update_full_text_search, CREATE_FULL_TEXT_SEARCH, first_sync_finished
from lbry.wallet.server.db.trending import TRENDING_ALGORITHMS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -117,6 +116,15 @@ class SQLDB:
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
CREATE_LANGUAGE_TABLE = """
create table if not exists language (
language text not null,
claim_hash bytes not null,
height integer not null
);
create unique index if not exists language_claim_hash_language_idx on language (claim_hash, language);
"""
CREATE_CLAIMTRIE_TABLE = """
create table if not exists claimtrie (
normalized text primary key,
@@ -174,12 +182,18 @@ class SQLDB:
for tag_value, tag_key in COMMON_TAGS.items()
)
LANGUAGE_INDEXES = '\n'.join(
f"create unique index if not exists language_{language}_idx on language (language, claim_hash) WHERE language='{language}';"
for language in INDEXED_LANGUAGES
)
CREATE_TABLES_QUERY = (
CREATE_CLAIM_TABLE +
CREATE_FULL_TEXT_SEARCH +
CREATE_SUPPORT_TABLE +
CREATE_CLAIMTRIE_TABLE +
CREATE_TAG_TABLE
CREATE_TAG_TABLE +
CREATE_LANGUAGE_TABLE
)
def __init__(
@@ -305,7 +319,7 @@ class SQLDB:
self.execute('commit;')
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
claim_hashes, claims, tags = set(), [], {}
claim_hashes, claims, tags, languages = set(), [], {}, {}
for txo in txos:
tx = txo.tx_ref.tx
@@ -316,6 +330,13 @@ class SQLDB:
#self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.")
continue
language = None
try:
if txo.claim.is_stream and txo.claim.stream.languages:
language = txo.claim.stream.languages[0].language
except:
pass
claim_hash = txo.claim_hash
claim_hashes.add(claim_hash)
claim_record = {
@@ -373,6 +394,9 @@ class SQLDB:
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
if language:
languages[(language, claim_hash)] = (language, claim_hash, tx.height)
for tag in clean_tags(claim.message.tags):
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
@@ -383,6 +407,10 @@ class SQLDB:
self.executemany(
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
)
if languages:
self.executemany(
"INSERT OR IGNORE INTO language (language, claim_hash, height) VALUES (?, ?, ?)", languages.values()
)
return claims
@@ -262,6 +262,17 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
await self.assertFindsClaims([claim5], fee_currency='usd')
async def test_search_by_language(self):
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
claim2 = await self.stream_create('claim2', fee_amount='0.9', fee_currency='lbc')
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc', languages='en')
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc', languages='en')
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd', languages='es')
await self.assertFindsClaims([claim4, claim3], any_languages=['en'])
await self.assertFindsClaims([claim5], any_languages=['es'])
await self.assertFindsClaims([claim5, claim4, claim3], any_languages=['en', 'es'])
await self.assertFindsClaims([], fee_currency='foo')
async def test_search_by_channel(self):
@@ -319,6 +330,25 @@ class ClaimSearchCommand(ClaimTestCase):
not_channel_ids=[chan2_id], has_channel_signature=True, valid_channel_signature=True)
await match([], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True, valid_channel_signature=True)
async def test_limit_claims_per_channel(self):
match = self.assertFindsClaims
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
claim1 = await self.stream_create('claim1')
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
claim3 = await self.stream_create('claim3', channel_id=chan1_id)
claim4 = await self.stream_create('claim4', channel_id=chan1_id)
claim5 = await self.stream_create('claim5', channel_id=chan2_id)
claim6 = await self.stream_create('claim6', channel_id=chan2_id)
await match(
[claim6, claim5, claim4, claim3, claim1],
limit_claims_per_channel=2, claim_type='stream'
)
await match(
[claim6, claim5, claim4, claim3, claim2, claim1],
limit_claims_per_channel=3, claim_type='stream'
)
async def test_claim_type_and_media_type_search(self):
# create an invalid/unknown claim
address = await self.account.receiving.get_or_create_usable_address()
+300 -23
View File
@@ -1,5 +1,4 @@
import re
import time
import typing
from math import ceil
@@ -30,6 +29,17 @@ class MockedCommentServer:
'timestamp': None,
'channel_url': None,
'is_hidden': False,
'is_pinned': False,
}
REACT_SCHEMA = {
'react_id': None,
'comment_id': None,
'reaction_type': None,
'timestamp': None,
'signature': None,
'channel_id': None,
'channel_name': None
}
def __init__(self, port=2903):
@@ -39,7 +49,9 @@ class MockedCommentServer:
self.runner = None
self.server = None
self.comments = []
self.reacts = {}
self.comment_id = 0
self.react_id = 0
@classmethod
def _create_comment(cls, **kwargs):
@@ -47,6 +59,12 @@ class MockedCommentServer:
schema.update(**kwargs)
return schema
@classmethod
def _react(cls, **kwargs):
schema = cls.REACT_SCHEMA.copy()
schema.update(**kwargs)
return schema
def create_comment(self, claim_id=None, parent_id=None, channel_name=None, channel_id=None, **kwargs):
comment_id = self.comment_id
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
@@ -105,6 +123,21 @@ class MockedCommentServer:
return True
return False
def pin_comment(
self,
comment_id: typing.Union[int, str],
channel_name: str, channel_id: str, remove: bool,
signing_ts: str, signature: str
):
comment_id = self.get_comment_id(comment_id)
if self.is_signable(signature, signing_ts):
if remove:
self.comments[comment_id]['is_pinned'] = False
else:
self.comments[comment_id]['is_pinned'] = True
return self.comments[comment_id]
return False
def hide_comments(self, pieces: list):
hidden = []
for p in pieces:
@@ -117,27 +150,18 @@ class MockedCommentServer:
'visible': list(comment_ids - set(hidden))
}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
def get_claim_comments(self, claim_id, hidden=False, visible=False, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
if hidden ^ visible:
comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(comments)/page_size),
'total_items': len(comments),
'items': [self.clean(c) for c in (comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], comments)))
}
def get_claim_hidden_comments(self, claim_id, hidden=True, page=1, page_size=50):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
select_comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(select_comments) / page_size),
'total_items': len(select_comments),
'items': [self.clean(c) for c in (select_comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda c: c['is_hidden'], comments)))
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], self.comments)))
}
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
@@ -157,15 +181,99 @@ class MockedCommentServer:
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
def react(
self,
comment_ids=None,
channel_name=None,
channel_id=None,
remove=None,
clear_types=None,
type=None,
signing_ts=None,
**kwargs
):
comment_batch = comment_ids.split(',')
for item in comment_batch:
react_id = self.react_id
c_id = item.strip()
reacts_for_comment_id = [r for r in list(self.reacts.values()) if r['comment_id'] == c_id]
channels_reacts_for_comment_id = [r for r in reacts_for_comment_id if r['channel_id'] == channel_id]
if remove:
matching_react = None
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == type:
matching_react = reaction
break
if matching_react:
self.reacts.pop(int(matching_react['react_id']), None)
else:
if clear_types:
for r_type in clear_types.split(','):
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == r_type:
x = self.reacts.pop(int(reaction['react_id']), None)
react = self._react(
react_id=str(react_id),
comment_id=str(c_id),
channel_name=channel_name,
channel_id=channel_id,
reaction_type=type,
timestamp=str(int(time.time())),
**kwargs
)
self.reacts[self.react_id] = react
self.react_id += 1
self.clean(react)
return True
def list_reacts(self, comment_ids, channel_id, channel_name, types=None, **kwargs):
all_types = list(set([r['reaction_type'] for r in list(self.reacts.values())]))
comment_id_list = comment_ids.split(',')
# _reacts: {'a1': {'like': 0, 'dislike': 0}, 'a2': {'like': 0, 'dislike': 0}}
own_reacts = {}
other_reacts = {}
# for each comment_id
# add comment_id: {} to own_reacts and other_reacts
# for each react in own_reacts
# for each react in other_reacts
for cid in comment_id_list:
own_reacts[cid] = {}
other_reacts[cid] = {}
for r_type in all_types:
own_reacts[cid][r_type] = 0
other_reacts[cid][r_type] = 0
# process that comment ids reactions for own and other categories
reacts_for_comment = list(filter(lambda c: c['comment_id'] == cid, list(self.reacts.values())))
if types:
reacts_for_comment = list(filter(lambda c: c['reaction_type'] in types.split(','), reacts_for_comment))
own_reacts_for_comment = list(filter(lambda c: c['channel_id'] == channel_id, reacts_for_comment))
other_reacts_for_comment = list(filter(lambda c: c['channel_id'] != channel_id, reacts_for_comment))
if own_reacts_for_comment:
for react in own_reacts_for_comment:
own_reacts[cid][react['reaction_type']] += 1
if other_reacts_for_comment:
for react in other_reacts_for_comment:
other_reacts[cid][react['reaction_type']] += 1
return {
'my_reactions': own_reacts,
'others_reactions': other_reacts,
}
methods = {
'get_claim_comments': get_claim_comments,
'comment.List': get_claim_comments,
'get_comments_by_id': get_comments_by_id,
'create_comment': create_comment,
'abandon_comment': abandon_comment,
'get_channel_from_comment_id': get_comment_channel_by_id,
'get_claim_hidden_comments': get_claim_hidden_comments,
'hide_comments': hide_comments,
'edit_comment': edit_comment,
'comment.Create': create_comment,
'comment.Abandon': abandon_comment,
'comment.GetChannelFromCommentID': get_comment_channel_by_id,
'comment.Hide': hide_comments,
'comment.Pin': pin_comment,
'comment.Edit': edit_comment,
'reaction.React': react,
'reaction.List': list_reacts,
}
def process_json(self, body) -> dict:
@@ -427,6 +535,56 @@ class CommentCommands(CommandTestCase):
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test05_comment_pin(self):
# wherein a bee makes a sick burn on moth's channel and moth pins it
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
moth_stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
moth_claim_id = moth_stream['outputs'][0]['claim_id']
comment1 = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=moth_claim_id,
channel_id=bee['claim_id']
)
self.assertFalse(comment1['is_pinned'])
comment2 = await self.daemon.jsonrpc_comment_create(
comment='sick burn, brah',
claim_id=moth_claim_id,
channel_id=moth_id,
)
self.assertFalse(comment2['is_pinned'])
comments = await self.daemon.jsonrpc_comment_list(moth_claim_id)
comments_items = comments['items']
self.assertIn('is_pinned', comments_items[0])
self.assertFalse(comments_items[0]['is_pinned'])
pinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name']
)
self.assertTrue(pinned['is_pinned'])
comments_after_pinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_pin = comments_after_pinning['items']
self.assertTrue(items_after_pin[1]['is_pinned'])
unpinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
remove=True
)
self.assertFalse(unpinned['is_pinned'])
comments_after_unpinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_unpin = comments_after_unpinning['items']
self.assertFalse(items_after_unpin[1]['is_pinned'])
async def test06_comment_list(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
@@ -450,7 +608,6 @@ class CommentCommands(CommandTestCase):
channel_id=bee['claim_id']
)
all_comments = [other_comment, owner_comment, hidden_comment]
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
@@ -532,3 +689,123 @@ class CommentCommands(CommandTestCase):
comment='If you see it and you mean then you know you have to go',
comment_id=original_cid
)
async def test06_reactions(self):
# wherein a bee insulted by a rude moth accidentally
# 1) likes an insult
# 2) clicks dislike instead, testing "clear_types"
# 3) then clicks dislike again, testing "remove"
# calls from the point of view of the moth
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
claim_id = stream['outputs'][0]['claim_id']
first_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth['claim_id'],
)
second_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
all_comments = [second_comment, first_comment]
comment_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(comment_list)
)
self.assertEqual(comment_list['total_items'], 2)
bee_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='like',
)
moth_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_type='like',
)
reactions = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1}}, 'others_reactions': {'0': {'like': 1}}}
self.assertEqual(reactions['my_reactions']['0']['like'], 1)
self.assertEqual(reactions['others_reactions']['0']['like'], 1)
bee_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='dislike',
clear_types='like',
)
reactions_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1, 'dislike', 0}}, 'others_reactions': {'0': {'like': 0, 'dislike': 1}}}
self.assertEqual(reactions_after_bee_dislikes['my_reactions']['0']['like'], 1)
self.assertEqual(reactions_after_bee_dislikes['my_reactions']['0']['dislike'], 0)
self.assertEqual(reactions_after_bee_dislikes['others_reactions']['0']['dislike'], 1)
self.assertEqual(reactions_after_bee_dislikes['others_reactions']['0']['like'], 0)
only_likes_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_types='like',
)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions']['0']['like'], 1)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions']['0']['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions']['0']['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions']['0']['like'], 0)
bee_un_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
remove=True,
react_type='dislike',
)
reactions_after_bee_absconds = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_bee_absconds['my_reactions']['0']['like'], 1)
self.assertNotIn('dislike', reactions_after_bee_absconds['my_reactions']['0'])
self.assertEqual(reactions_after_bee_absconds['others_reactions']['0']['like'], 0)
self.assertNotIn('dislike', reactions_after_bee_absconds['others_reactions']['0'])
bee_reacts_to_both_comments = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'] + ',' + second_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='frozen_tom',
)
reactions_after_double_frozen_tom = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'] + ',' + second_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_double_frozen_tom['my_reactions']['0']['like'], 1)
self.assertNotIn('dislike', reactions_after_double_frozen_tom['my_reactions']['0'])
self.assertEqual(reactions_after_double_frozen_tom['others_reactions']['0']['frozen_tom'], 1)
self.assertEqual(reactions_after_double_frozen_tom['others_reactions']['1']['frozen_tom'], 1)
+5 -5
View File
@@ -37,8 +37,8 @@ class TestSigningComments(AsyncioTestCase):
rswanson = await get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
sign_comment(comment_body, rswanson, sign_comment_id=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, sign_comment_id=True))
async def test04_invalid_signature(self):
rswanson = await get_channel('@RonSwanson')
@@ -53,7 +53,7 @@ class TestSigningComments(AsyncioTestCase):
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
fake_abandon_signal = chair_comment.copy()
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
sign_comment(fake_abandon_signal, jeanralphio, sign_comment_id=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, sign_comment_id=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, sign_comment_id=True))
@@ -1,7 +1,9 @@
import binascii
import unittest
from lbry.dht.error import DecodeError
from lbry.dht.serialization.bencoding import _bencode
from lbry.dht.serialization.datagram import RequestDatagram, ResponseDatagram, decode_datagram, ErrorDatagram
from lbry.dht.serialization.datagram import _decode_datagram
from lbry.dht.serialization.datagram import REQUEST_TYPE, RESPONSE_TYPE, ERROR_TYPE
from lbry.dht.serialization.datagram import make_compact_address, decode_compact_address
@@ -139,6 +141,38 @@ class TestDatagram(unittest.TestCase):
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_str_or_int_keys(self):
datagram = decode_datagram(_bencode({
b'0': 0,
b'1': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
b'2': b'111111111111111111111111111111111111111111111111',
b'3': b'ping',
b'4': [{b'protocolVersion': 1}],
b'5': b'should not error'
}))
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_mixed_str_or_int_keys(self):
# datagram, _ = _bencode({
# b'0': 0,
# 1: b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
# b'2': b'111111111111111111111111111111111111111111111111',
# 3: b'ping',
# b'4': [{b'protocolVersion': 1}],
# b'5': b'should not error'
# }))
encoded = binascii.unhexlify(b"64313a3069306569316532303a0abcb5269d6cfc1e87a08e920bf39fe9df8e92fc313a3234383a313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131693365343a70696e67313a346c6431353a70726f746f636f6c56657273696f6e6931656565313a3531363a73686f756c64206e6f74206572726f7265")
self.assertDictEqual(
{
'packet_type': 0,
'rpc_id': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
'node_id': b'111111111111111111111111111111111111111111111111',
'method': b'ping',
'args': [{b'protocolVersion': 1}]
}, _decode_datagram(encoded)[0]
)
class TestCompactAddress(unittest.TestCase):
def test_encode_decode(self, address='1.2.3.4', port=4444, node_id=b'1' * 48):
+13 -6
View File
@@ -40,7 +40,7 @@ class TestReflector(AsyncioTestCase):
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream_manager.config.reflect_streams = False
self.stream = await self.stream_manager.create(file_path)
async def _test_reflect_stream(self, response_chunk_size):
@@ -93,8 +93,10 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
stop.set()
# this used to raise (and then propagate) a CancelledError
self.assertListEqual(await reflect_task, [])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertFalse(self.stream.is_fully_reflected)
self.assertFalse(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
async def test_result_from_disconnect_after_sd_transfer(self):
stop = asyncio.Event()
@@ -112,7 +114,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -134,7 +137,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -156,7 +160,9 @@ class TestReflector(AsyncioTestCase):
await not_incoming.wait()
await incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
# self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
@@ -181,7 +187,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await self.stream_manager.delete(self.stream, delete_file=True)
# this used to raise OSError when it can't read the deleted blob for the upload
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()