Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
148983516e | ||
|
|
c96d1d9c32 | ||
|
|
7c7a0d4bdf | ||
|
|
cc829a7bf4 | ||
|
|
e0ea6383e2 | ||
|
|
bcec5dc2ae | ||
|
|
cba9c16a06 | ||
|
|
dd68fb077b | ||
|
|
c2294e97db | ||
|
|
c0f512ace7 | ||
|
|
3305eb67c6 | ||
|
|
c9d637b4da | ||
|
|
ae3e8fadf5 | ||
|
|
a1abd94387 | ||
|
|
9b463a8cab | ||
|
|
babc54a240 | ||
|
|
5836a93b21 | ||
|
|
557348e345 | ||
|
|
9adfec6b00 | ||
|
|
3a496902f8 |
No files matched your search
@@ -1,6 +1,9 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
|
||||
from prometheus_client import Counter, Gauge
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.node import Node
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
@@ -9,6 +12,15 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
announcements_sent_metric = Counter(
|
||||
"announcements_sent", "Number of announcements sent and their respective status.", namespace="dht_node",
|
||||
labelnames=("peers", "error"),
|
||||
)
|
||||
announcement_queue_size_metric = Gauge(
|
||||
"announcement_queue_size", "Number of hashes waiting to be announced.", namespace="dht_node",
|
||||
labelnames=("scope",)
|
||||
)
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
@@ -18,12 +30,15 @@ class BlobAnnouncer:
|
||||
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
self.announcements_sent_metric.labels(peers=peers, error=False).inc()
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
self.announcements_sent_metric.labels(peers=0, error=True).inc()
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
@@ -37,6 +52,7 @@ class BlobAnnouncer:
|
||||
log.warning("No peers in DHT, announce round skipped")
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
self.announcement_queue_size_metric.labels(scope="global").set(len(self.announce_queue))
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue) > 0:
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
CACHE_SIZE = 2048
|
||||
CACHE_SIZE = 16384
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -37,7 +37,8 @@ from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.error import (
|
||||
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
|
||||
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError,
|
||||
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError
|
||||
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError,
|
||||
InputValueError
|
||||
)
|
||||
from lbry.extras import system_info
|
||||
from lbry.extras.daemon import analytics
|
||||
@@ -3614,15 +3615,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(existing_claims) != 1:
|
||||
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
raise InputValueError(
|
||||
f"Can't find the stream '{claim_id}' in account(s) {account_ids}."
|
||||
)
|
||||
|
||||
old_txo = existing_claims[0]
|
||||
if not old_txo.claim.is_stream:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"A claim with id '{claim_id}' was found but it is not a stream claim."
|
||||
if not old_txo.claim.is_stream and not old_txo.claim.is_repost:
|
||||
# in principle it should work with any type of claim, but its safer to
|
||||
# limit it to ones we know won't be broken. in the future we can expand
|
||||
# this if we have a test case for e.g. channel or support claims
|
||||
raise InputValueError(
|
||||
f"A claim with id '{claim_id}' was found but it is not a stream or repost claim."
|
||||
)
|
||||
|
||||
if bid is not None:
|
||||
@@ -3653,28 +3656,32 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if replace:
|
||||
claim = Claim()
|
||||
if old_txo.claim.stream.has_source:
|
||||
claim.stream.message.source.CopyFrom(
|
||||
old_txo.claim.stream.message.source
|
||||
)
|
||||
stream_type = old_txo.claim.stream.stream_type
|
||||
if stream_type:
|
||||
old_stream_type = getattr(old_txo.claim.stream.message, stream_type)
|
||||
new_stream_type = getattr(claim.stream.message, stream_type)
|
||||
new_stream_type.CopyFrom(old_stream_type)
|
||||
claim.stream.update(file_path=file_path, **kwargs)
|
||||
if old_txo.claim.is_stream:
|
||||
if old_txo.claim.stream.has_source:
|
||||
claim.stream.message.source.CopyFrom(
|
||||
old_txo.claim.stream.message.source
|
||||
)
|
||||
stream_type = old_txo.claim.stream.stream_type
|
||||
if stream_type:
|
||||
old_stream_type = getattr(old_txo.claim.stream.message, stream_type)
|
||||
new_stream_type = getattr(claim.stream.message, stream_type)
|
||||
new_stream_type.CopyFrom(old_stream_type)
|
||||
else:
|
||||
claim = Claim.from_bytes(old_txo.claim.to_bytes())
|
||||
|
||||
if old_txo.claim.is_stream:
|
||||
claim.stream.update(file_path=file_path, **kwargs)
|
||||
|
||||
if clear_channel:
|
||||
claim.clear_signature()
|
||||
tx = await Transaction.claim_update(
|
||||
old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0],
|
||||
channel if not clear_channel else None
|
||||
)
|
||||
|
||||
new_txo = tx.outputs[0]
|
||||
stream_hash = None
|
||||
if not preview:
|
||||
if not preview and old_txo.claim.is_stream:
|
||||
old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash)
|
||||
old_stream = old_stream[0] if old_stream else None
|
||||
if file_path is not None:
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import filetype
|
||||
import logging
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
@@ -166,10 +168,38 @@ types_map = {
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
}
|
||||
|
||||
# maps detected extensions to the possible analogs
|
||||
# i.e. .cbz file is actually a .zip
|
||||
synonyms_map = {
|
||||
'.zip': ['.cbz'],
|
||||
'.rar': ['.cbr'],
|
||||
'.ar': ['.a']
|
||||
}
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
extension = ext.strip().lower()
|
||||
|
||||
try:
|
||||
kind = filetype.guess(path)
|
||||
if kind:
|
||||
real_extension = f".{kind.extension}"
|
||||
|
||||
if extension != real_extension:
|
||||
if extension:
|
||||
log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}")
|
||||
else:
|
||||
log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}")
|
||||
|
||||
if extension not in synonyms_map.get(real_extension, []):
|
||||
extension = real_extension
|
||||
|
||||
except OSError as error:
|
||||
pass
|
||||
|
||||
if extension[1:]:
|
||||
if extension in types_map:
|
||||
return types_map[extension]
|
||||
|
||||
@@ -39,9 +39,14 @@ class DeterministicChannelKeyManager:
|
||||
self.account = account
|
||||
self.last_known = 0
|
||||
self.cache = {}
|
||||
self.private_key: Optional[PrivateKey] = None
|
||||
if account.private_key is not None:
|
||||
self.private_key = account.private_key.child(KeyPath.CHANNEL)
|
||||
self._private_key: Optional[PrivateKey] = None
|
||||
|
||||
@property
|
||||
def private_key(self):
|
||||
if self._private_key is None:
|
||||
if self.account.private_key is not None:
|
||||
self._private_key = self.account.private_key.child(KeyPath.CHANNEL)
|
||||
return self._private_key
|
||||
|
||||
def maybe_generate_deterministic_key_for_channel(self, txo):
|
||||
if self.private_key is None:
|
||||
|
||||
@@ -2,6 +2,7 @@ NULL_HASH32 = b'\x00'*32
|
||||
|
||||
CENT = 1000000
|
||||
COIN = 100*CENT
|
||||
DUST = 1000
|
||||
|
||||
TIMEOUT = 30.0
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
__hub_url__ = (
|
||||
"https://github.com/lbryio/hub/releases/download/v0.2021.12.18.1/hub"
|
||||
"https://github.com/lbryio/hub/releases/download/v0.2022.01.21.1/hub"
|
||||
)
|
||||
from .node import Conductor
|
||||
from .service import ConductorService
|
||||
@@ -14,7 +14,7 @@ from lbry.schema.purchase import Purchase
|
||||
from lbry.schema.support import Support
|
||||
|
||||
from .script import InputScript, OutputScript
|
||||
from .constants import COIN, NULL_HASH32
|
||||
from .constants import COIN, DUST, NULL_HASH32
|
||||
from .bcd_data_stream import BCDataStream
|
||||
from .hash import TXRef, TXRefImmutable
|
||||
from .util import ReadOnlyList
|
||||
@@ -818,10 +818,10 @@ class Transaction:
|
||||
)
|
||||
if payment > cost:
|
||||
change = payment - cost
|
||||
if change > cost_of_change:
|
||||
change_amount = change - cost_of_change
|
||||
if change_amount > DUST:
|
||||
change_address = await change_account.change.get_or_create_usable_address()
|
||||
change_hash160 = change_account.ledger.address_to_hash160(change_address)
|
||||
change_amount = change - cost_of_change
|
||||
change_output = Output.pay_pubkey_hash(change_amount, change_hash160)
|
||||
change_output.is_internal_transfer = True
|
||||
tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)])
|
||||
|
||||
@@ -56,7 +56,8 @@ setup(
|
||||
'attrs==18.2.0',
|
||||
'pylru==1.1.0',
|
||||
'elasticsearch==7.10.1',
|
||||
'grpcio==1.38.0'
|
||||
'grpcio==1.38.0',
|
||||
'filetype==1.0.9'
|
||||
] + PLYVEL,
|
||||
extras_require={
|
||||
'torrent': ['lbry-libtorrent'],
|
||||
|
||||
@@ -365,6 +365,13 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
self.assertEqual(daemon2.jsonrpc_wallet_status(),
|
||||
{'is_locked': False, 'is_encrypted': True, 'is_syncing': False})
|
||||
|
||||
async def test_locking_unlocking_does_not_break_deterministic_channels(self):
|
||||
self.assertTrue(self.daemon.jsonrpc_wallet_encrypt("password"))
|
||||
self.assertTrue(self.daemon.jsonrpc_wallet_lock())
|
||||
self.account.deterministic_channel_keys._private_key = None
|
||||
self.assertTrue(self.daemon.jsonrpc_wallet_unlock("password"))
|
||||
await self.channel_create()
|
||||
|
||||
async def test_sync_with_encryption_and_password_change(self):
|
||||
daemon, daemon2 = self.daemon, self.daemon2
|
||||
wallet, wallet2 = daemon.wallet_manager.default_wallet, daemon2.wallet_manager.default_wallet
|
||||
|
||||
@@ -1499,6 +1499,11 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search)
|
||||
self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff')
|
||||
|
||||
await self.stream_update(repost_id, bid='0.42')
|
||||
searched_repost = (await self.claim_search(claim_id=repost_id))[0]
|
||||
self.assertEqual(searched_repost['amount'], '0.42')
|
||||
self.assertEqual(searched_repost['signing_channel']['claim_id'], spam_claim_id)
|
||||
|
||||
async def test_filtering_channels_for_removing_content(self):
|
||||
await self.channel_create('@some_channel', '0.1')
|
||||
await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good'])
|
||||
@@ -2054,6 +2059,17 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(txs[0]['fee'], '-0.000107')
|
||||
await self.assertBalance(self.account, '9.9795695')
|
||||
|
||||
async def test_no_source_and_valid_channel_signature_and_media_type(self):
|
||||
await self.channel_create('@spam2', '1.0')
|
||||
await self.stream_create('bar', '1.0', channel_name='@spam2', preview=True, confirm=False, file_path=self.video_file_name)
|
||||
no_source_claims = await self.claim_search(has_source=False, valid_channel_signature=True,
|
||||
media_type="video/mp4")
|
||||
mp4_claims = await self.claim_search(valid_channel_signature=True, media_type="video/mp4")
|
||||
no_source_claims_no_media = await self.claim_search(has_source=False, valid_channel_signature=True)
|
||||
self.assertEqual(0, len(no_source_claims))
|
||||
self.assertEqual(1, len(mp4_claims))
|
||||
self.assertEqual(0, len(no_source_claims_no_media))
|
||||
|
||||
async def test_abandoning_stream_at_loss(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
tx = await self.stream_create(bid='0.0001')
|
||||
|
||||
@@ -3,6 +3,17 @@ from lbry.testcase import CommandTestCase
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
async def test_txo_dust_prevention(self):
|
||||
address = await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
tx = await self.account_send('9.9997758', address)
|
||||
# dust prevention threshold not reached, small txo created
|
||||
self.assertEqual(2, len(tx['outputs']))
|
||||
self.assertEqual(tx['outputs'][1]['amount'], '0.0001002')
|
||||
tx = await self.account_send('9.999706', address)
|
||||
# dust prevention prevented dust
|
||||
self.assertEqual(1, len(tx['outputs']))
|
||||
self.assertEqual(tx['outputs'][0]['amount'], '9.999706')
|
||||
|
||||
async def test_transaction_show(self):
|
||||
# local tx
|
||||
result = await self.out(self.daemon.jsonrpc_account_send(
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import unittest
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
|
||||
class MediaTypeTests(unittest.TestCase):
|
||||
def test_guess_media_type_from_path_only(self):
|
||||
kind = guess_media_type('/tmp/test.mkv')
|
||||
self.assertEqual(kind, ('video/x-matroska', 'video'))
|
||||
|
||||
def test_defaults_for_no_extension(self):
|
||||
kind = guess_media_type('/tmp/test')
|
||||
self.assertEqual(kind, ('application/octet-stream', 'binary'))
|
||||
|
||||
def test_defaults_for_unknown_extension(self):
|
||||
kind = guess_media_type('/tmp/test.unk')
|
||||
self.assertEqual(kind, ('application/x-ext-unk', 'binary'))
|
||||
|
||||
def test_spoofed_unknown(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_unknown.txt')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_lz4 = bytearray([0x04,0x22,0x4d,0x18])
|
||||
fd.write(bytes_lz4)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/x-ext-lz4', 'binary'))
|
||||
|
||||
def test_spoofed_known(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_known.avi')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_zip = bytearray([0x50,0x4b,0x03,0x06])
|
||||
fd.write(bytes_zip)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/zip', 'binary'))
|
||||
|
||||
def test_spoofed_synonym(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_known.cbz')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_zip = bytearray([0x50,0x4b,0x03,0x06])
|
||||
fd.write(bytes_zip)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/vnd.comicbook+zip', 'document'))
|
||||
Reference in new issue
Block a user