Compare commits

..
15 changed files with 113 additions and 44 deletions

No files matched your search

+1 -1
View File
@@ -97,7 +97,7 @@ jobs:
strategy:
matrix:
os:
- ubuntu-18.04
- ubuntu-16.04
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
+2 -2
View File
@@ -15,8 +15,8 @@ jobs:
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
- uses: slackapi/slack-github-action@v1.14.0
env:
CHANGELOG: ${{ toJSON(steps.markdown.outputs.text) }}
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
with:
payload: '{"type": "mrkdwn", "text": ${{ env.CHANGELOG }}}'
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }}'
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.101.2"
__version__ = "0.101.1"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+2 -2
View File
@@ -156,9 +156,9 @@ class KademliaPeer:
if not len(self._node_id) == constants.HASH_LENGTH:
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
raise ValueError("invalid udp port")
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
raise ValueError("invalid tcp port")
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
if not is_valid_public_ipv4(self.address, self.allow_localhost):
raise ValueError(f"invalid ip address: '{self.address}'")
+5 -1
View File
@@ -452,7 +452,11 @@ class KademliaProtocol(DatagramProtocol):
try:
peer = self.routing_table.get_peer(request_datagram.node_id)
except IndexError:
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
except ValueError as err:
log.warning("error replying to %s: %s", address[0], str(err))
return
try:
self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
+2 -1
View File
@@ -53,10 +53,11 @@ Code | Name | Message
407 | DataDownload | Failed to download blob. *generic*
410 | Resolve | Failed to resolve '{url}'.
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
420 | KeyFeeAboveMaxAllowed | {message}
421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
423 | TooManyClaimSearchParameters | {key} cant be set for more than {limit} items.
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
+8
View File
@@ -242,6 +242,14 @@ class IncompatibleWalletServerError(WalletError):
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
class TooManyClaimSearchParametersError(WalletError):
def __init__(self, key, limit):
self.key = key
self.limit = limit
super().__init__(f"{key} cant be set for more than {limit} items.")
class ServerPaymentInvalidAddressError(WalletError):
def __init__(self, address):
+22
View File
@@ -190,6 +190,10 @@ def get_argument_parser():
'--no-logging', dest='no_logging', action="store_true",
help='Disable all logging of any kind.'
)
start.add_argument(
'--show-stack-trace', dest='show_stack_trace', action="store_true",
help='Show full stack trace on error.'
)
start.add_argument(
'--verbose', nargs="*",
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
@@ -255,10 +259,28 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
logger.getChild('lbry').setLevel(logging.DEBUG)
def make_stack_trace_supressing_record_factory(original_factory):
def factory(*args, **kwargs):
record = original_factory(*args, **kwargs)
if record.exc_info:
record.msg += f" ({record.exc_info[0].__name__}: {record.exc_info[1]})"
record.exc_info = None
record.exc_text = None
return record
return factory
def warning_error_handler(loop, context):
logging.warning("Uncaught exception: %s", context)
def run_daemon(args: argparse.Namespace, conf: Config):
loop = asyncio.get_event_loop()
if args.verbose is not None:
loop.set_debug(True)
elif not args.show_stack_trace:
loop.set_exception_handler(warning_error_handler)
logging.setLogRecordFactory(make_stack_trace_supressing_record_factory(logging.getLogRecordFactory()))
if not args.no_logging:
setup_logging(logging.getLogger(), args, conf)
daemon = Daemon(conf)
-3
View File
@@ -191,9 +191,6 @@ class WalletManager:
'data_path': config.wallet_dir,
'tx_cache_size': config.transaction_cache_size
}
if 'LBRY_FEE_PER_NAME_CHAR' in os.environ:
ledger_config['fee_per_name_char'] = int(os.environ.get('LBRY_FEE_PER_NAME_CHAR'))
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
if not os.path.exists(wallets_directory):
@@ -10,7 +10,7 @@ from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
from elasticsearch.helpers import async_streaming_bulk
from lbry.crypto.base58 import Base58
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError
from lbry.schema.result import Outputs, Censor
from lbry.schema.tags import clean_tags
from lbry.schema.url import URL, normalize_name
@@ -465,6 +465,8 @@ def expand_query(**kwargs):
for key, value in kwargs.items():
key = key.replace('claim.', '')
many = key.endswith('__in') or isinstance(value, list)
if many and len(value) > 2048:
raise TooManyClaimSearchParametersError(key, 2048)
if many:
key = key.replace('__in', '')
value = list(filter(None, value))
+20 -10
View File
@@ -22,19 +22,29 @@ async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
db.row_factory = namedtuple_factory
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
for num, claim in enumerate(db.execute(f"""
SELECT claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash)
WHERE claim.height % {shards_total} = {shard_num}
ORDER BY claim.height desc
SELECT claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
cr.has_source as reposted_has_source,
cr.claim_type as reposted_claim_type,
cr.stream_type as reposted_stream_type,
cr.media_type as reposted_media_type,
cr.duration as reposted_duration,
cr.fee_amount as reposted_fee_amount,
cr.fee_currency as reposted_fee_currency,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
WHERE claim.height % {shards_total} = {shard_num}
ORDER BY claim.height desc
""")):
claim = dict(claim._asdict())
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
claim['censor_type'] = 0
claim['censoring_channel_id'] = None
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
+13 -3
View File
@@ -818,10 +818,15 @@ class SQLDB:
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
cr.has_source as reposted_has_source,
cr.claim_type as reposted_claim_type,
cr.stream_type as reposted_stream_type,
cr.media_type as reposted_media_type,
cr.duration as reposted_duration,
cr.fee_amount as reposted_fee_amount,
cr.fee_currency as reposted_fee_currency,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash)
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
"""
for claim in self.execute(query):
@@ -830,6 +835,11 @@ class SQLDB:
claim['censor_type'] = 0
censoring_channel_hash = None
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
for reason_id in id_set:
if reason_id in self.blocked_streams:
claim['censor_type'] = 2
+8 -3
View File
@@ -15,14 +15,13 @@ from asyncio import Event, sleep
from collections import defaultdict
from functools import partial
from binascii import hexlify
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from elasticsearch import ConnectionTimeout
from prometheus_client import Counter, Info, Histogram, Gauge
import lbry
from lbry.utils import LRUCacheWithMetrics
from lbry.error import TooManyClaimSearchParametersError
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYLevelDB
@@ -1028,7 +1027,13 @@ class LBRYElectrumX(SessionBase):
async def claimtrie_search(self, **kwargs):
if kwargs:
return await self.run_and_cache_query('search', kwargs)
try:
return await self.run_and_cache_query('search', kwargs)
except TooManyClaimSearchParametersError as err:
await asyncio.sleep(2)
self.logger.warning("Got an invalid query from %s, for %s with more than %d elements.",
self.peer_address()[0], err.key, err.limit)
return RPCError(1, str(err))
async def claimtrie_resolve(self, *urls):
if urls:
@@ -304,13 +304,15 @@ class ClaimSearchCommand(ClaimTestCase):
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc')
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc')
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd')
repost1 = await self.stream_repost(self.get_claim_id(claim1), 'repost1')
repost5 = await self.stream_repost(self.get_claim_id(claim5), 'repost5')
await self.assertFindsClaims([claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
await self.assertFindsClaims([claim4, claim3, claim2, claim1], fee_currency='lbc')
await self.assertFindsClaims([claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
await self.assertFindsClaims([repost5, repost1, claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
await self.assertFindsClaims([repost1, claim4, claim3, claim2, claim1], fee_currency='lbc')
await self.assertFindsClaims([repost1, claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
await self.assertFindsClaims([claim5], fee_currency='usd')
await self.assertFindsClaims([repost5, claim5], fee_currency='usd')
async def test_search_by_language(self):
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
@@ -476,7 +478,8 @@ class ClaimSearchCommand(ClaimTestCase):
octet = await self.stream_create()
video = await self.stream_create('chrome', file_path=self.video_file_name)
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
repost = await self.stream_repost(self.get_claim_id(image))
image_repost = await self.stream_repost(self.get_claim_id(image), 'image-repost')
video_repost = await self.stream_repost(self.get_claim_id(video), 'video-repost')
collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)])
channel = await self.channel_create()
unknown = self.sout(tx)
@@ -484,25 +487,25 @@ class ClaimSearchCommand(ClaimTestCase):
# claim_type
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
await self.assertFindsClaims([channel], claim_type='channel')
await self.assertFindsClaims([repost], claim_type='repost')
await self.assertFindsClaims([video_repost, image_repost], claim_type='repost')
await self.assertFindsClaims([collection], claim_type='collection')
# stream_type
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
await self.assertFindsClaims([video], stream_types=['video'])
await self.assertFindsClaims([image], stream_types=['image'])
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
await self.assertFindsClaims([video_repost, video], stream_types=['video'])
await self.assertFindsClaims([image_repost, image], stream_types=['image'])
await self.assertFindsClaims([video_repost, image_repost, image, video], stream_types=['video', 'image'])
# media_type
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
await self.assertFindsClaims([video], media_types=['video/mp4'])
await self.assertFindsClaims([image], media_types=['image/png'])
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
await self.assertFindsClaims([video_repost, video], media_types=['video/mp4'])
await self.assertFindsClaims([image_repost, image], media_types=['image/png'])
await self.assertFindsClaims([video_repost, image_repost, image, video], media_types=['video/mp4', 'image/png'])
# duration
await self.assertFindsClaim(video, duration='>14')
await self.assertFindsClaim(video, duration='<16')
await self.assertFindsClaim(video, duration=15)
await self.assertFindsClaims([video_repost, video], duration='>14')
await self.assertFindsClaims([video_repost, video], duration='<16')
await self.assertFindsClaims([video_repost, video], duration=15)
await self.assertFindsClaims([], duration='>100')
await self.assertFindsClaims([], duration='<14')
@@ -4,6 +4,7 @@ import lbry
import lbry.wallet
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
from lbry.wallet.network import ClientSession
from lbry.wallet.rpc import RPCError
from lbry.wallet.server.db.elasticsearch.sync import run as run_sync, make_es_index
from lbry.wallet.server.session import LBRYElectrumX
from lbry.testcase import IntegrationTestCase, CommandTestCase
@@ -191,7 +192,7 @@ class TestHubDiscovery(CommandTestCase):
)
class TestStressFlush(CommandTestCase):
class TestStress(CommandTestCase):
async def test_flush_over_66_thousand(self):
history = self.conductor.spv_node.server.db.history
history.flush_count = 66_000
@@ -199,3 +200,9 @@ class TestStressFlush(CommandTestCase):
self.assertEqual(history.flush_count, 66_001)
await self.generate(1)
self.assertEqual(history.flush_count, 66_002)
async def test_thousands_claim_ids_on_search(self):
await self.stream_create()
with self.assertRaises(RPCError) as err:
await self.claim_search(not_channel_ids=[("%040x" % i) for i in range(8196)])
self.assertEqual(err.exception.message, 'not_channel_ids cant be set for more than 2048 items.')