Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
265eb9da67 | ||
|
|
21c71bfac1 | ||
|
|
6baaed3581 | ||
|
|
152dbfd5d1 | ||
|
|
a56d14086b | ||
|
|
aee87693f8 | ||
|
|
976b4affd9 | ||
|
|
e222b6ad9c | ||
|
|
19b17374e8 | ||
|
|
43989122bb | ||
|
|
72712d6047 | ||
|
|
0b52d2cc15 | ||
|
|
8304102136 | ||
|
|
3381aefcfa | ||
|
|
279a365cb1 | ||
|
|
2c9e00da56 |
No files matched your search
@@ -15,8 +15,8 @@ jobs:
|
||||
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
|
||||
- uses: slackapi/slack-github-action@v1.14.0
|
||||
env:
|
||||
CHANGELOG: ${{ toJSON(steps.markdown.outputs.text) }}
|
||||
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
|
||||
with:
|
||||
payload: '{"type": "mrkdwn", "text": ${{ env.CHANGELOG }}}'
|
||||
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }}'
|
||||
|
||||
+2
-2
@@ -156,9 +156,9 @@ class KademliaPeer:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
|
||||
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
|
||||
@@ -452,7 +452,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
except ValueError as err:
|
||||
log.warning("error replying to %s: %s", address[0], str(err))
|
||||
return
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
|
||||
@@ -53,10 +53,11 @@ Code | Name | Message
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
423 | TooManyClaimSearchParameters | {key} cant be set for more than {limit} items.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
|
||||
@@ -242,6 +242,14 @@ class IncompatibleWalletServerError(WalletError):
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class TooManyClaimSearchParametersError(WalletError):
|
||||
|
||||
def __init__(self, key, limit):
|
||||
self.key = key
|
||||
self.limit = limit
|
||||
super().__init__(f"{key} cant be set for more than {limit} items.")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
|
||||
@@ -190,6 +190,10 @@ def get_argument_parser():
|
||||
'--no-logging', dest='no_logging', action="store_true",
|
||||
help='Disable all logging of any kind.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--show-stack-trace', dest='show_stack_trace', action="store_true",
|
||||
help='Show full stack trace on error.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
|
||||
@@ -255,10 +259,28 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def make_stack_trace_supressing_record_factory(original_factory):
|
||||
def factory(*args, **kwargs):
|
||||
record = original_factory(*args, **kwargs)
|
||||
if record.exc_info:
|
||||
record.msg += f" ({record.exc_info[0].__name__}: {record.exc_info[1]})"
|
||||
record.exc_info = None
|
||||
record.exc_text = None
|
||||
return record
|
||||
return factory
|
||||
|
||||
|
||||
def warning_error_handler(loop, context):
|
||||
logging.warning("Uncaught exception: %s", context)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
elif not args.show_stack_trace:
|
||||
loop.set_exception_handler(warning_error_handler)
|
||||
logging.setLogRecordFactory(make_stack_trace_supressing_record_factory(logging.getLogRecordFactory()))
|
||||
if not args.no_logging:
|
||||
setup_logging(logging.getLogger(), args, conf)
|
||||
daemon = Daemon(conf)
|
||||
|
||||
@@ -10,7 +10,7 @@ from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
|
||||
from elasticsearch.helpers import async_streaming_bulk
|
||||
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
|
||||
from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError
|
||||
from lbry.schema.result import Outputs, Censor
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
@@ -465,6 +465,8 @@ def expand_query(**kwargs):
|
||||
for key, value in kwargs.items():
|
||||
key = key.replace('claim.', '')
|
||||
many = key.endswith('__in') or isinstance(value, list)
|
||||
if many and len(value) > 2048:
|
||||
raise TooManyClaimSearchParametersError(key, 2048)
|
||||
if many:
|
||||
key = key.replace('__in', '')
|
||||
value = list(filter(None, value))
|
||||
|
||||
@@ -22,19 +22,29 @@ async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
|
||||
db.row_factory = namedtuple_factory
|
||||
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
|
||||
for num, claim in enumerate(db.execute(f"""
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
""")):
|
||||
claim = dict(claim._asdict())
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_id'] = None
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
|
||||
@@ -818,10 +818,15 @@ class SQLDB:
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
|
||||
"""
|
||||
for claim in self.execute(query):
|
||||
@@ -830,6 +835,11 @@ class SQLDB:
|
||||
claim['censor_type'] = 0
|
||||
censoring_channel_hash = None
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
for reason_id in id_set:
|
||||
if reason_id in self.blocked_streams:
|
||||
claim['censor_type'] = 2
|
||||
|
||||
@@ -15,14 +15,13 @@ from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from binascii import hexlify
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
from elasticsearch import ConnectionTimeout
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.error import TooManyClaimSearchParametersError
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
@@ -1028,7 +1027,13 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
try:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
except TooManyClaimSearchParametersError as err:
|
||||
await asyncio.sleep(2)
|
||||
self.logger.warning("Got an invalid query from %s, for %s with more than %d elements.",
|
||||
self.peer_address()[0], err.key, err.limit)
|
||||
return RPCError(1, str(err))
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
if urls:
|
||||
|
||||
@@ -304,13 +304,15 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc')
|
||||
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc')
|
||||
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd')
|
||||
repost1 = await self.stream_repost(self.get_claim_id(claim1), 'repost1')
|
||||
repost5 = await self.stream_repost(self.get_claim_id(claim5), 'repost5')
|
||||
|
||||
await self.assertFindsClaims([claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
|
||||
await self.assertFindsClaims([claim4, claim3, claim2, claim1], fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
|
||||
await self.assertFindsClaims([repost5, repost1, claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
|
||||
await self.assertFindsClaims([repost1, claim4, claim3, claim2, claim1], fee_currency='lbc')
|
||||
await self.assertFindsClaims([repost1, claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim5], fee_currency='usd')
|
||||
await self.assertFindsClaims([repost5, claim5], fee_currency='usd')
|
||||
|
||||
async def test_search_by_language(self):
|
||||
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
|
||||
@@ -476,7 +478,8 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
octet = await self.stream_create()
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
|
||||
repost = await self.stream_repost(self.get_claim_id(image))
|
||||
image_repost = await self.stream_repost(self.get_claim_id(image), 'image-repost')
|
||||
video_repost = await self.stream_repost(self.get_claim_id(video), 'video-repost')
|
||||
collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)])
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
@@ -484,25 +487,25 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
await self.assertFindsClaims([repost], claim_type='repost')
|
||||
await self.assertFindsClaims([video_repost, image_repost], claim_type='repost')
|
||||
await self.assertFindsClaims([collection], claim_type='collection')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
await self.assertFindsClaims([video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
await self.assertFindsClaims([video_repost, video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image_repost, image], stream_types=['image'])
|
||||
await self.assertFindsClaims([video_repost, image_repost, image, video], stream_types=['video', 'image'])
|
||||
|
||||
# media_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
|
||||
await self.assertFindsClaims([video_repost, video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image_repost, image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([video_repost, image_repost, image, video], media_types=['video/mp4', 'image/png'])
|
||||
|
||||
# duration
|
||||
await self.assertFindsClaim(video, duration='>14')
|
||||
await self.assertFindsClaim(video, duration='<16')
|
||||
await self.assertFindsClaim(video, duration=15)
|
||||
await self.assertFindsClaims([video_repost, video], duration='>14')
|
||||
await self.assertFindsClaims([video_repost, video], duration='<16')
|
||||
await self.assertFindsClaims([video_repost, video], duration=15)
|
||||
await self.assertFindsClaims([], duration='>100')
|
||||
await self.assertFindsClaims([], duration='<14')
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.rpc import RPCError
|
||||
from lbry.wallet.server.db.elasticsearch.sync import run as run_sync, make_es_index
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
@@ -191,7 +192,7 @@ class TestHubDiscovery(CommandTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestStressFlush(CommandTestCase):
|
||||
class TestStress(CommandTestCase):
|
||||
async def test_flush_over_66_thousand(self):
|
||||
history = self.conductor.spv_node.server.db.history
|
||||
history.flush_count = 66_000
|
||||
@@ -199,3 +200,9 @@ class TestStressFlush(CommandTestCase):
|
||||
self.assertEqual(history.flush_count, 66_001)
|
||||
await self.generate(1)
|
||||
self.assertEqual(history.flush_count, 66_002)
|
||||
|
||||
async def test_thousands_claim_ids_on_search(self):
|
||||
await self.stream_create()
|
||||
with self.assertRaises(RPCError) as err:
|
||||
await self.claim_search(not_channel_ids=[("%040x" % i) for i in range(8196)])
|
||||
self.assertEqual(err.exception.message, 'not_channel_ids cant be set for more than 2048 items.')
|
||||
Reference in new issue
Block a user